From 06a3dd763932559e3d127a71e827182383e22b48 Mon Sep 17 00:00:00 2001 From: Jesus Armando Anaya Date: Sat, 8 Aug 2026 06:26:54 -0700 Subject: [PATCH] feat(inference): SAM 2 point-prompt adapter and suggest route (#424 slice 3a) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Everything the editor's suggest gesture needs server-side: an adapter that serves PointPrompt, the mask-to-geometry conversion, and the route the editor will call. The port carries no notion of what a class allows, deliberately — widening it would push a project's schema into a protocol that has to be implementable by a hosted service. So the adapter answers with the most informative shape it has and the orchestration narrows it to the geometry kinds the caller names. Adapter resolution reads the model's own declared family rather than the connection kind, which says only where a model runs. A connection pointed at a detector and asked with points is then refused with the port's vocabulary instead of failing inside a forward pass. The embedding cache and the provider pool ship together because each is defeated by the absence of the other: a provider rebuilt per request carries an empty cache, so every click would pay the encode. The ImageFormat media-type table moves into the kernel domain, since the inference adapters became its second reader and a route may not be imported from there. cf. #418, #421, #424 --- docs/inference.md | 64 ++++ frontend/ui-core/src/generated/api.ts | 305 +++++++++++++++++++ frontend/ui-core/src/generated/checks.ts | 19 ++ openapi.json | 370 +++++++++++++++++++++++ src/visionset/inference/__init__.py | 114 +++---- src/visionset/inference/cache.py | 111 +++++++ src/visionset/inference/masks.py | 285 +++++++++++++++++ src/visionset/inference/providers.py | 196 ++++++++++++ src/visionset/inference/sam_provider.py | 305 +++++++++++++++++++ src/visionset/inference/suggestions.py | 121 ++++++++ src/visionset/kernel/domain/__init__.py | 6 + src/visionset/kernel/domain/media.py | 32 ++ src/visionset/server/models.py | 68 +++++ src/visionset/server/routes/__init__.py | 1 + src/visionset/server/routes/assets.py | 35 +-- src/visionset/server/routes/inference.py | 68 ++++- tests/inference/stubs.py | 182 +++++++++++ tests/inference/test_cache.py | 90 ++++++ tests/inference/test_masks.py | 169 +++++++++++ tests/inference/test_providers.py | 207 +++++++++++++ tests/inference/test_sam_provider.py | 213 +++++++++++++ tests/server/test_downloads.py | 14 +- tests/server/test_suggest.py | 320 ++++++++++++++++++++ 23 files changed, 3188 insertions(+), 107 deletions(-) create mode 100644 src/visionset/inference/cache.py create mode 100644 src/visionset/inference/masks.py create mode 100644 src/visionset/inference/providers.py create mode 100644 src/visionset/inference/sam_provider.py create mode 100644 src/visionset/inference/suggestions.py create mode 100644 tests/inference/stubs.py create mode 100644 tests/inference/test_cache.py create mode 100644 tests/inference/test_masks.py create mode 100644 tests/inference/test_providers.py create mode 100644 tests/inference/test_sam_provider.py create mode 100644 tests/server/test_suggest.py diff --git a/docs/inference.md b/docs/inference.md index d3a8a2df..4c474a1a 100644 --- a/docs/inference.md +++ b/docs/inference.md @@ -124,6 +124,70 @@ Half precision (`fp16`, `float16`, `half` — the spelling is yours) applies on it is not the conservative choice it looks like: `float16` arithmetic outside CUDA's autocast is slower than the `float32` it was avoiding. +## Suggesting a shape from a click + +A connection whose model answers *places* rather than *words* can propose a shape for whatever +sits under a point. One call, one asset, one set of points: + +```http +POST /inference/suggest +{ + "project_id": "…", "asset_id": "…", "connection_id": "…", + "positive": [{"x": 412.0, "y": 233.0}], + "negative": [], + "allowed_geometries": ["polygon"] +} +``` + +```json +{ + "model_ref": "some/segmenter@abc123", + "region": { + "geometry": {"type": "polygon", "points": [[404.0, 221.0], …]}, + "confidence": 0.87 + } +} +``` + +**Points are in the asset's own pixels**, the same frame every geometry in a project uses. +`positive` says *this*; `negative` says *not that*, which is how an over-eager first answer gets +carved back without starting over. Refining means sending the accumulated points again, not a +diff — the call keeps no state about your gesture, so the same points always answer the same way. + +**`allowed_geometries` is your schema, not a preference.** The answer comes back in one of the +kinds you named or not at all: name `polygon` and you get the outline; name only `bbox` and you +get that outline's extent; name a kind that holds no shape and `region` is `null`. Answering in a +kind your schema would refuse would hand you a suggestion that cannot be accepted. + +**`region: null` is a successful answer with nothing to propose** — a click on empty background, +a model less sure than you asked for, or a shape too thin to be a polygon. `model_ref` is still +there, because it is what an accepted suggestion has to carry. + +**`detail` controls how much of the outline survives simplification**, as a fraction of the +region's own size rather than a pixel count, so one setting works on a thing eight pixels across +and a thing eight hundred across alike. Omit it and the server's default keeps a typical object in +the 10–40 vertex range. Smaller is more faithful and more vertices. + +### Nothing is written, and the first click is the slow one + +Asking is not annotating. The response is a proposal: turning it into an annotation is an ordinary +annotation write that carries `provenance: model`, this response's `model_ref` and its +`confidence`. Discarding it costs nothing, because nothing was recorded. + +A segmenter reads the whole image once and then answers any number of clicks from that reading, so +the **first** call for an asset pays for the encode and the ones after it do not. That cached +reading is the only thing the call leaves behind — an optimisation, not a record. It lives in the +server process, is bounded, and a restart costs you nothing but the latency of the next first +click. + +### When it refuses + +The connection is resolved before the asset, deliberately: if you are part-way through setting a +connection up, you should hear about the connection rather than about an asset that was never the +problem. A connection whose weights are not here yet is `INFERENCE_CONNECTION_NOT_SET_UP` and +names `download` as the remedy; one whose model answers words rather than places is +`UNSUPPORTED_PROMPT`. + ## What a connection is not It is **not a credential store**, yet. An HTTP connection carries no secret today, and the field diff --git a/frontend/ui-core/src/generated/api.ts b/frontend/ui-core/src/generated/api.ts index 7eb1e4ee..e8da2a02 100644 --- a/frontend/ui-core/src/generated/api.ts +++ b/frontend/ui-core/src/generated/api.ts @@ -796,6 +796,55 @@ export interface paths { patch?: never; trace?: never; }; + "/inference/suggest": { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + get?: never; + put?: never; + /** + * Suggest Region + * @description Propose a shape for the thing under those points. + * + * The server side of the editor's suggest gesture (`cf. #424`). One asset, one + * prompt set, one answer — batch prediction is a separate path and is not this + * one. + * + * **Nothing is written and nothing is remembered.** A suggestion is a proposal: + * accepting it is a later, ordinary annotation write carrying `provenance: + * model`, this response's `model_ref`, and its `confidence`. Discarding it + * costs a request that already finished. The only thing that outlives the call + * is a cached image embedding, which is an optimisation rather than a record — + * so the same points sent twice answer the same way, and a restart changes + * nothing but the latency of the first click. + * + * **The first click on an asset is the slow one.** A segmenter reads the whole + * image once and then answers any number of clicks from that reading almost for + * free, which is what makes refining by adding points practical. Sending the + * accumulated points — rather than a diff — is what keeps this stateless. + * + * **`allowed_geometries` is the caller's schema, not a preference.** The answer + * is produced in one of the kinds named or not at all: a class that admits + * polygons gets the outline, a class that admits only boxes gets its extent, + * and a class that admits neither gets `region: null`. Answering in a kind the + * schema would refuse would produce a suggestion that cannot be accepted. + * + * A null `region` is a successful answer with nothing to propose. Refusals are + * reserved for things the caller can act on: an unknown project, asset or + * connection is 404; a connection whose weights are not here yet, or whose kind + * this build cannot run, is 409 and names what to do; a connection whose model + * answers words rather than places is 422. + */ + post: operations["suggest_region"]; + delete?: never; + options?: never; + head?: never; + patch?: never; + trace?: never; + }; "/ingest-jobs/{job_id}": { parameters: { query?: never; @@ -2334,6 +2383,30 @@ export interface components { /** Y */ y: number; }; + /** + * BboxGeometry + * @description An axis-aligned rectangle: top-left corner plus size. + * + * ``width`` and ``height`` must be strictly positive — a zero-area box is as + * meaningless as a negative one, so neither is accepted. ``x`` and ``y`` are + * unconstrained: an annotation may legitimately start outside the asset's + * bounds when an object is clipped by the frame edge. + */ + BboxGeometry: { + /** Height */ + height: number; + /** + * @description discriminator enum property added by openapi-typescript + * @enum {string} + */ + type: "bbox"; + /** Width */ + width: number; + /** X */ + x: number; + /** Y */ + y: number; + }; /** Body_register_image_source */ Body_register_image_source: { /** @@ -2438,6 +2511,21 @@ export interface components { */ type: "classification_tag"; }; + /** + * ClassificationGeometry + * @description A whole-asset tag: the annotation carries a class but no coordinates. + * + * It exists as a variant rather than as ``geometry: None`` so that every + * annotation has a geometry with a discriminator, and so the union stays the + * single place that answers "what shape is this label?". + */ + ClassificationGeometry: { + /** + * @description discriminator enum property added by openapi-typescript + * @enum {string} + */ + type: "classification_tag"; + }; /** * ConnectionAction * @description What can be asked of an inference connection. Order is display order. @@ -2911,6 +2999,27 @@ export interface components { */ type: "polygon"; }; + /** + * PolygonGeometry + * @description A closed polygon, as at least three ``(x, y)`` vertices. + * + * The closing edge is implicit: the last point joins the first, and repeating + * the first point at the end is NOT expected. Self-intersection is not + * validated — M1 accepts any ring of three or more points, and rejecting + * degenerate shapes is left to a later milestone. + */ + PolygonGeometry: { + /** Points */ + points: [ + number, + number + ][]; + /** + * @description discriminator enum property added by openapi-typescript + * @enum {string} + */ + type: "polygon"; + }; /** * PolylineBody * @description An open path of at least two points, in order. Nothing joins the ends. @@ -2927,6 +3036,45 @@ export interface components { */ type: "polyline"; }; + /** + * PolylineGeometry + * @description An open path, as at least two ``(x, y)`` vertices in order. + * + * The contrast with :class:`PolygonGeometry` is the whole definition: a polygon + * is a *ring* whose closing edge is implicit, and a polyline is a *path* whose + * ends stay apart. Nothing joins the last point to the first, and a caller that + * repeats the first point at the end has drawn a closed path — which is a legal + * polyline, and not the same value as the polygon with those vertices. + * + * **The order of the points is the geometry**, not an incidental detail of how + * they were collected. A lane runs from one end to the other, and reversing the + * list is a different annotation of the same pixels. There is nothing to + * validate in that — an ordered sequence is ordered — which is worth saying + * because the ordering rule a lane *format* wants is a different rule: TuSimple + * requires points sorted by ascending Y, and :mod:`visionset.formats.lanes` + * enforces that at the boundary where it applies. Putting it here would make one + * format's invariant a condition of storing a lane at all, and would refuse + * every horizontal path in a domain that has no idea what a road is. + * + * Degeneracy is refused in exactly one case, the analogue of the zero-area box + * :class:`BboxGeometry` already declines: a path whose points are all the same + * point has no length and describes nothing. Consecutive duplicates within a + * longer path are left alone — they arrive from real digitizers and from honest + * resampling, and they cost a renderer nothing — and self-intersection is not + * validated here for the same reason it is not validated for a polygon. + */ + PolylineGeometry: { + /** Points */ + points: [ + number, + number + ][]; + /** + * @description discriminator enum property added by openapi-typescript + * @enum {string} + */ + type: "polyline"; + }; /** * ProgressCounts * @description How many assets sit in each annotation state. @@ -3271,6 +3419,85 @@ export interface components { /** Val */ val: number; }; + /** + * SuggestPoint + * @description One click, in the asset's own pixel coordinates. + * + * An object rather than a two-element array because a JSON ``[x, y]`` is a + * shape a generated client types as ``number[]`` and a reader has to guess the + * order of. The domain's own tuples are fine — Python has positional meaning — + * but the wire is read by people. + */ + SuggestPoint: { + /** X */ + x: number; + /** Y */ + y: number; + }; + /** + * SuggestRequest + * @description Where somebody clicked, on what, through which connection. + * + * Everything travels in the body rather than in the path: the call names an + * asset *and* a connection, and neither owns the other. Putting one in the path + * would make it look like the parent of the request, which is how a URL is + * read. + */ + SuggestRequest: { + /** Allowed Geometries */ + allowed_geometries: components["schemas"]["GeometryType"][]; + /** + * Asset Id + * Format: uuid + */ + asset_id: string; + /** + * Connection Id + * Format: uuid + */ + connection_id: string; + /** Detail */ + detail?: number | null; + /** Negative */ + negative?: components["schemas"]["SuggestPoint"][]; + /** Positive */ + positive: components["schemas"]["SuggestPoint"][]; + /** + * Project Id + * Format: uuid + */ + project_id: string; + }; + /** + * SuggestedRegion + * @description One proposed shape and how sure the model is of it. + */ + SuggestedRegion: { + /** Confidence */ + confidence: number; + /** Geometry */ + geometry: components["schemas"]["BboxGeometry"] | components["schemas"]["PolygonGeometry"] | components["schemas"]["PolylineGeometry"] | components["schemas"]["ClassificationGeometry"]; + }; + /** + * SuggestionOut + * @description What the model proposes, or an honest nothing. + * + * ``region`` is null when there is no suggestion, and that is an ordinary + * answer rather than an error: a click can land on sky, the model can be less + * sure than the caller asked for, and the shape found can be one this class + * cannot hold. A 404 or a 409 for any of those would be telling the caller they + * did something wrong when they did not. + * + * ``model_ref`` is echoed on every answer, including the empty one, because it + * is what an accepted suggestion has to carry into its annotation — and a + * caller that had to remember which connection it asked would be keeping a + * second copy of something the response can simply state. + */ + SuggestionOut: { + /** Model Ref */ + model_ref: string; + region?: components["schemas"]["SuggestedRegion"] | null; + }; /** * VideoProvenanceOut * @description What a clip turned out to be, and the rate it is decomposed at. @@ -5480,6 +5707,84 @@ export interface operations { }; }; }; + suggest_region: { + parameters: { + query?: never; + header?: never; + path?: never; + cookie?: never; + }; + requestBody: { + content: { + "application/json": components["schemas"]["SuggestRequest"]; + }; + }; + responses: { + /** @description Successful Response */ + 200: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["SuggestionOut"]; + }; + }; + /** @description Missing or invalid bearer token */ + 401: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ErrorBody"]; + }; + }; + /** @description No such resource */ + 404: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ErrorBody"]; + }; + }; + /** @description The resource's state refuses this request */ + 409: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ErrorBody"]; + }; + }; + /** @description The request payload is not processable */ + 422: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ErrorBody"]; + }; + }; + /** @description Unhandled server error, with an incident id */ + 500: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ErrorBody"]; + }; + }; + /** @description The workspace is busy; retry after the header says */ + 503: { + headers: { + [name: string]: unknown; + }; + content: { + "application/json": components["schemas"]["ErrorBody"]; + }; + }; + }; + }; get_ingest_job: { parameters: { query?: never; diff --git a/frontend/ui-core/src/generated/checks.ts b/frontend/ui-core/src/generated/checks.ts index 93b6bfa9..959cc2da 100644 --- a/frontend/ui-core/src/generated/checks.ts +++ b/frontend/ui-core/src/generated/checks.ts @@ -245,6 +245,24 @@ export const checkSourcePage: Check = export const checkSplitAssignmentOut: Check = /*#__PURE__*/ object({ "test": [true, arrayOf(isString)], "train": [true, arrayOf(isString)], "val": [true, arrayOf(isString)] } as const); +export const checkBboxGeometry: Check = + /*#__PURE__*/ object({ "height": [true, isNumber], "type": [true, lit("bbox")], "width": [true, isNumber], "x": [true, isNumber], "y": [true, isNumber] } as const); + +export const checkClassificationGeometry: Check = + /*#__PURE__*/ object({ "type": [true, lit("classification_tag")] } as const); + +export const checkPolygonGeometry: Check = + /*#__PURE__*/ object({ "points": [true, arrayOf(tuple([isNumber, isNumber] as const))], "type": [true, lit("polygon")] } as const); + +export const checkPolylineGeometry: Check = + /*#__PURE__*/ object({ "points": [true, arrayOf(tuple([isNumber, isNumber] as const))], "type": [true, lit("polyline")] } as const); + +export const checkSuggestedRegion: Check = + /*#__PURE__*/ object({ "confidence": [true, isNumber], "geometry": [true, tagged("type", { "bbox": checkBboxGeometry, "classification_tag": checkClassificationGeometry, "polygon": checkPolygonGeometry, "polyline": checkPolylineGeometry })] } as const); + +export const checkSuggestionOut: Check = + /*#__PURE__*/ object({ "model_ref": [true, isString], "region": [false, either([checkSuggestedRegion, isNull] as const)] } as const); + // One alias per operation. `unwrap` takes these, never a schema check directly, so that // `tests/scripts/checks_wiring.test.mjs` can pair every call with its own operationId. @@ -320,6 +338,7 @@ export const checkSetAssetProgress = checkAssetProgressOut; export const checkStartBatch = checkBatchOut; export const checkStartIngest = checkIngestJobOut; export const checkStartJob = checkJobOut; +export const checkSuggestRegion = checkSuggestionOut; export const checkUpdateAnnotations = checkAnnotationPage; export const checkUpdateInferenceConnection = checkConnectionOut; export const checkVerifyRelease = checkReleaseVerificationOut; diff --git a/openapi.json b/openapi.json index ea0fb224..8814f860 100644 --- a/openapi.json +++ b/openapi.json @@ -1237,6 +1237,44 @@ "title": "BboxBody", "type": "object" }, + "BboxGeometry": { + "additionalProperties": false, + "description": "An axis-aligned rectangle: top-left corner plus size.\n\n``width`` and ``height`` must be strictly positive \u2014 a zero-area box is as\nmeaningless as a negative one, so neither is accepted. ``x`` and ``y`` are\nunconstrained: an annotation may legitimately start outside the asset's\nbounds when an object is clipped by the frame edge.", + "properties": { + "height": { + "exclusiveMinimum": 0.0, + "title": "Height", + "type": "number" + }, + "type": { + "const": "bbox", + "default": "bbox", + "title": "Type", + "type": "string" + }, + "width": { + "exclusiveMinimum": 0.0, + "title": "Width", + "type": "number" + }, + "x": { + "title": "X", + "type": "number" + }, + "y": { + "title": "Y", + "type": "number" + } + }, + "required": [ + "x", + "y", + "width", + "height" + ], + "title": "BboxGeometry", + "type": "object" + }, "Body_register_image_source": { "properties": { "files": { @@ -1440,6 +1478,20 @@ "title": "ClassificationBody", "type": "object" }, + "ClassificationGeometry": { + "additionalProperties": false, + "description": "A whole-asset tag: the annotation carries a class but no coordinates.\n\nIt exists as a variant rather than as ``geometry: None`` so that every\nannotation has a geometry with a discriminator, and so the union stays the\nsingle place that answers \"what shape is this label?\".", + "properties": { + "type": { + "const": "classification_tag", + "default": "classification_tag", + "title": "Type", + "type": "string" + } + }, + "title": "ClassificationGeometry", + "type": "object" + }, "ConnectionAction": { "description": "What can be asked of an inference connection. Order is display order.", "enum": [ @@ -2394,6 +2446,41 @@ "title": "PolygonBody", "type": "object" }, + "PolygonGeometry": { + "additionalProperties": false, + "description": "A closed polygon, as at least three ``(x, y)`` vertices.\n\nThe closing edge is implicit: the last point joins the first, and repeating\nthe first point at the end is NOT expected. Self-intersection is not\nvalidated \u2014 M1 accepts any ring of three or more points, and rejecting\ndegenerate shapes is left to a later milestone.", + "properties": { + "points": { + "items": { + "maxItems": 2, + "minItems": 2, + "prefixItems": [ + { + "type": "number" + }, + { + "type": "number" + } + ], + "type": "array" + }, + "minItems": 3, + "title": "Points", + "type": "array" + }, + "type": { + "const": "polygon", + "default": "polygon", + "title": "Type", + "type": "string" + } + }, + "required": [ + "points" + ], + "title": "PolygonGeometry", + "type": "object" + }, "PolylineBody": { "additionalProperties": false, "description": "An open path of at least two points, in order. Nothing joins the ends.", @@ -2428,6 +2515,41 @@ "title": "PolylineBody", "type": "object" }, + "PolylineGeometry": { + "additionalProperties": false, + "description": "An open path, as at least two ``(x, y)`` vertices in order.\n\nThe contrast with :class:`PolygonGeometry` is the whole definition: a polygon\nis a *ring* whose closing edge is implicit, and a polyline is a *path* whose\nends stay apart. Nothing joins the last point to the first, and a caller that\nrepeats the first point at the end has drawn a closed path \u2014 which is a legal\npolyline, and not the same value as the polygon with those vertices.\n\n**The order of the points is the geometry**, not an incidental detail of how\nthey were collected. A lane runs from one end to the other, and reversing the\nlist is a different annotation of the same pixels. There is nothing to\nvalidate in that \u2014 an ordered sequence is ordered \u2014 which is worth saying\nbecause the ordering rule a lane *format* wants is a different rule: TuSimple\nrequires points sorted by ascending Y, and :mod:`visionset.formats.lanes`\nenforces that at the boundary where it applies. Putting it here would make one\nformat's invariant a condition of storing a lane at all, and would refuse\nevery horizontal path in a domain that has no idea what a road is.\n\nDegeneracy is refused in exactly one case, the analogue of the zero-area box\n:class:`BboxGeometry` already declines: a path whose points are all the same\npoint has no length and describes nothing. Consecutive duplicates within a\nlonger path are left alone \u2014 they arrive from real digitizers and from honest\nresampling, and they cost a renderer nothing \u2014 and self-intersection is not\nvalidated here for the same reason it is not validated for a polygon.", + "properties": { + "points": { + "items": { + "maxItems": 2, + "minItems": 2, + "prefixItems": [ + { + "type": "number" + }, + { + "type": "number" + } + ], + "type": "array" + }, + "minItems": 2, + "title": "Points", + "type": "array" + }, + "type": { + "const": "polyline", + "default": "polyline", + "title": "Type", + "type": "string" + } + }, + "required": [ + "points" + ], + "title": "PolylineGeometry", + "type": "object" + }, "ProgressCounts": { "description": "How many assets sit in each annotation state.", "properties": { @@ -3151,6 +3273,157 @@ "title": "SplitRecipeBody", "type": "object" }, + "SuggestPoint": { + "description": "One click, in the asset's own pixel coordinates.\n\nAn object rather than a two-element array because a JSON ``[x, y]`` is a\nshape a generated client types as ``number[]`` and a reader has to guess the\norder of. The domain's own tuples are fine \u2014 Python has positional meaning \u2014\nbut the wire is read by people.", + "properties": { + "x": { + "title": "X", + "type": "number" + }, + "y": { + "title": "Y", + "type": "number" + } + }, + "required": [ + "x", + "y" + ], + "title": "SuggestPoint", + "type": "object" + }, + "SuggestRequest": { + "description": "Where somebody clicked, on what, through which connection.\n\nEverything travels in the body rather than in the path: the call names an\nasset *and* a connection, and neither owns the other. Putting one in the path\nwould make it look like the parent of the request, which is how a URL is\nread.", + "properties": { + "allowed_geometries": { + "items": { + "$ref": "#/components/schemas/GeometryType" + }, + "minItems": 1, + "title": "Allowed Geometries", + "type": "array" + }, + "asset_id": { + "format": "uuid", + "title": "Asset Id", + "type": "string" + }, + "connection_id": { + "format": "uuid", + "title": "Connection Id", + "type": "string" + }, + "detail": { + "anyOf": [ + { + "exclusiveMinimum": 0.0, + "maximum": 1.0, + "type": "number" + }, + { + "type": "null" + } + ], + "title": "Detail" + }, + "negative": { + "items": { + "$ref": "#/components/schemas/SuggestPoint" + }, + "title": "Negative", + "type": "array" + }, + "positive": { + "items": { + "$ref": "#/components/schemas/SuggestPoint" + }, + "minItems": 1, + "title": "Positive", + "type": "array" + }, + "project_id": { + "format": "uuid", + "title": "Project Id", + "type": "string" + } + }, + "required": [ + "project_id", + "asset_id", + "connection_id", + "positive", + "allowed_geometries" + ], + "title": "SuggestRequest", + "type": "object" + }, + "SuggestedRegion": { + "description": "One proposed shape and how sure the model is of it.", + "properties": { + "confidence": { + "maximum": 1.0, + "minimum": 0.0, + "title": "Confidence", + "type": "number" + }, + "geometry": { + "discriminator": { + "mapping": { + "bbox": "#/components/schemas/BboxGeometry", + "classification_tag": "#/components/schemas/ClassificationGeometry", + "polygon": "#/components/schemas/PolygonGeometry", + "polyline": "#/components/schemas/PolylineGeometry" + }, + "propertyName": "type" + }, + "oneOf": [ + { + "$ref": "#/components/schemas/BboxGeometry" + }, + { + "$ref": "#/components/schemas/PolygonGeometry" + }, + { + "$ref": "#/components/schemas/PolylineGeometry" + }, + { + "$ref": "#/components/schemas/ClassificationGeometry" + } + ], + "title": "Geometry" + } + }, + "required": [ + "geometry", + "confidence" + ], + "title": "SuggestedRegion", + "type": "object" + }, + "SuggestionOut": { + "description": "What the model proposes, or an honest nothing.\n\n``region`` is null when there is no suggestion, and that is an ordinary\nanswer rather than an error: a click can land on sky, the model can be less\nsure than the caller asked for, and the shape found can be one this class\ncannot hold. A 404 or a 409 for any of those would be telling the caller they\ndid something wrong when they did not.\n\n``model_ref`` is echoed on every answer, including the empty one, because it\nis what an accepted suggestion has to carry into its annotation \u2014 and a\ncaller that had to remember which connection it asked would be keeping a\nsecond copy of something the response can simply state.", + "properties": { + "model_ref": { + "title": "Model Ref", + "type": "string" + }, + "region": { + "anyOf": [ + { + "$ref": "#/components/schemas/SuggestedRegion" + }, + { + "type": "null" + } + ] + } + }, + "required": [ + "model_ref" + ], + "title": "SuggestionOut", + "type": "object" + }, "VideoProvenanceOut": { "description": "What a clip turned out to be, and the rate it is decomposed at.", "properties": { @@ -6149,6 +6422,103 @@ ] } }, + "/inference/suggest": { + "post": { + "description": "Propose a shape for the thing under those points.\n\nThe server side of the editor's suggest gesture (`cf. #424`). One asset, one\nprompt set, one answer \u2014 batch prediction is a separate path and is not this\none.\n\n**Nothing is written and nothing is remembered.** A suggestion is a proposal:\naccepting it is a later, ordinary annotation write carrying `provenance:\nmodel`, this response's `model_ref`, and its `confidence`. Discarding it\ncosts a request that already finished. The only thing that outlives the call\nis a cached image embedding, which is an optimisation rather than a record \u2014\nso the same points sent twice answer the same way, and a restart changes\nnothing but the latency of the first click.\n\n**The first click on an asset is the slow one.** A segmenter reads the whole\nimage once and then answers any number of clicks from that reading almost for\nfree, which is what makes refining by adding points practical. Sending the\naccumulated points \u2014 rather than a diff \u2014 is what keeps this stateless.\n\n**`allowed_geometries` is the caller's schema, not a preference.** The answer\nis produced in one of the kinds named or not at all: a class that admits\npolygons gets the outline, a class that admits only boxes gets its extent,\nand a class that admits neither gets `region: null`. Answering in a kind the\nschema would refuse would produce a suggestion that cannot be accepted.\n\nA null `region` is a successful answer with nothing to propose. Refusals are\nreserved for things the caller can act on: an unknown project, asset or\nconnection is 404; a connection whose weights are not here yet, or whose kind\nthis build cannot run, is 409 and names what to do; a connection whose model\nanswers words rather than places is 422.", + "operationId": "suggest_region", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SuggestRequest" + } + } + }, + "required": true + }, + "responses": { + "200": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/SuggestionOut" + } + } + }, + "description": "Successful Response" + }, + "401": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorBody" + } + } + }, + "description": "Missing or invalid bearer token" + }, + "404": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorBody" + } + } + }, + "description": "No such resource" + }, + "409": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorBody" + } + } + }, + "description": "The resource's state refuses this request" + }, + "422": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorBody" + } + } + }, + "description": "The request payload is not processable" + }, + "500": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorBody" + } + } + }, + "description": "Unhandled server error, with an incident id" + }, + "503": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ErrorBody" + } + } + }, + "description": "The workspace is busy; retry after the header says" + } + }, + "security": [ + { + "HTTPBearer": [] + } + ], + "summary": "Suggest Region", + "tags": [ + "inference" + ] + } + }, "/ingest-jobs/{job_id}": { "get": { "description": "Where a run is now.\n\n`processed` and `total` are written as the run goes, so this answers \"where\nis it\" rather than \"where did it end\". `total` is null for a clip \u2014 a video's\nframe count is a guess before extraction, so it is not reported.\n\nTerminal states are `completed` and `failed`. A `failed` job keeps its\ncounters exactly where they stopped, and `error` says why; unreadable\nindividual items are in `failures` and never fail a run on their own.", diff --git a/src/visionset/inference/__init__.py b/src/visionset/inference/__init__.py index 1f91917a..5d604da9 100644 --- a/src/visionset/inference/__init__.py +++ b/src/visionset/inference/__init__.py @@ -1,4 +1,4 @@ -# usage: from visionset.inference import provider_for, fetch_weights +# usage: from visionset.inference import provider_for, fetch_weights, suggest """The composition root for inference: a connection in, a ``ModelProvider`` out. **A sibling of ``visionset.formats``, ``visionset.wire`` and ``visionset.jobs``, @@ -18,106 +18,70 @@ runtime present. ``tests/architecture/test_optional_runtime.py`` proves it in a fresh interpreter. -**Resolution is by connection type, and there is no plugin registry.** #418's +**There is no plugin registry, and resolution happens in two steps.** #418's recorded decision is that adapters are instantiated from user-created model connections and never from a bundled default, which makes ``InferenceConnection`` the registry: a row somebody wrote, naming a kind, a model and where it runs. A provider discovered by entry point would have nothing to be instantiated *from*, and a workspace could acquire the ability to predict through an unrelated ``pip install`` — which is exactly what "VisionSet never downloads a model on its own" -exists to prevent. So the dispatch below is a ``match`` on two members, and it -grows by a deliberate change when a hosted adapter arrives. +exists to prevent. ``providers`` does the resolving: the connection's kind says +*where*, and the model's own config says *which family*, because a local +connection may hold a detector that answers words or a segmenter that answers +places and those are not interchangeable. + +**What each surface reaches for.** ``fetch_weights`` is the download, +``suggest`` is one click's worth of interactive segmentation, and ``provider_for`` +is the raw resolution underneath both. A surface serving clicks wants ``suggest`` +and the pool behind it; anything building a provider per call is paying a model +load per request, which is the latency failure D5 on #424 exists to prevent. """ from __future__ import annotations -from pathlib import Path - from visionset.inference._extra import EXTRA, INSTALL_COMMAND, MODULES, require +from visionset.inference.cache import ( + DEFAULT_EMBEDDING_CAPACITY, + DEFAULT_PROVIDER_CAPACITY, + BoundedCache, +) +from visionset.inference.masks import DEFAULT_DETAIL, narrowed, polygon_from from visionset.inference.nms import DEFAULT_IOU_THRESHOLD, suppressed +from visionset.inference.providers import ( + SEGMENTER_FAMILIES, + ProviderPool, + family_of, + provider_for, + resident, +) +from visionset.inference.sam_provider import LocalSamProvider +from visionset.inference.suggestions import suggest from visionset.inference.transformers_provider import LocalTransformersProvider from visionset.inference.weights import MODELS_DIRNAME, cache_root, download, fetch_weights -from visionset.kernel.domain import ConnectionSetupState, ConnectionType, InferenceConnection -from visionset.kernel.errors import ( - InferenceConnectionNotRunnable, - InferenceConnectionNotSetUp, -) -from visionset.kernel.ports import ModelProvider __all__ = [ + "DEFAULT_DETAIL", + "DEFAULT_EMBEDDING_CAPACITY", "DEFAULT_IOU_THRESHOLD", + "DEFAULT_PROVIDER_CAPACITY", "EXTRA", "INSTALL_COMMAND", "MODELS_DIRNAME", "MODULES", + "SEGMENTER_FAMILIES", + "BoundedCache", + "LocalSamProvider", "LocalTransformersProvider", + "ProviderPool", "cache_root", "download", + "family_of", "fetch_weights", + "narrowed", + "polygon_from", "provider_for", "require", + "resident", + "suggest", "suppressed", ] - - -def provider_for(connection: InferenceConnection, *, workspace_root: Path) -> ModelProvider: - """The thing that will answer for this connection, or the reason nothing can. - - Every refusal here is a ``VisionSetError`` carrying what happened and what to - do, never a stack trace and never a ``None`` a caller has to interpret — the - error contract, applied at the one place where "can this predict?" is finally - answered. - - Building one is cheap and loads no weights: a caller may construct a provider - to find out whether it *could* run, which is what makes these refusals worth - raising early. - - Raises: - InferenceConnectionNotSetUp: a local connection whose weights are not - here yet. The message names ``download_weights``, because that is the - action that makes the identical call succeed. - InferenceConnectionNotRunnable: nothing in this build runs a connection - of that kind. An ``http`` connection is well formed and unusable - here; the adapter that would speak to an endpoint is a later slice. - LocalInferenceUnavailable: the optional runtime is not installed. Raised - here rather than at the first ``predict`` so that a caller checking - usability gets the install command before it starts a batch. - """ - match connection.connection_type: - case ConnectionType.LOCAL: - return _local(connection, workspace_root=workspace_root) - case ConnectionType.HTTP: - raise InferenceConnectionNotRunnable( - f"connection {connection.name!r} is an http connection, and this build has no " - "adapter that can speak to one; use a local connection, or a later version" - ) - - -def _local(connection: InferenceConnection, *, workspace_root: Path) -> ModelProvider: - """A local provider, once both things it needs are true. - - The order of the two checks is deliberate. The connection's own state comes - first, because "your weights are not here" is about something the caller can - fix from where they are standing, while a missing extra is about the - installation and is the same answer for every connection in the workspace. - Reporting the machine's problem over the row's would tell somebody to run an - install when what they actually needed was a download. - """ - if connection.setup_state is not ConnectionSetupState.READY: - raise InferenceConnectionNotSetUp( - f"connection {connection.name!r} has no weights on this machine yet; " - "run its download_weights action first" - ) - require() - # ``device`` and ``precision`` are non-null on a local connection — the - # domain's cross-field rule is what makes that true — so the narrowing here - # is for the type checker rather than a possibility being handled. - assert connection.device is not None - return LocalTransformersProvider( - connection.model_id, - connection.model_revision, - device=connection.device, - precision=connection.precision, - cache_dir=cache_root(workspace_root), - connection_name=connection.name, - ) diff --git a/src/visionset/inference/cache.py b/src/visionset/inference/cache.py new file mode 100644 index 00000000..e42cb427 --- /dev/null +++ b/src/visionset/inference/cache.py @@ -0,0 +1,111 @@ +# usage: from visionset.inference.cache import BoundedCache +"""A least-recently-used cache, bounded by count, and the two capacities that use it. + +**Why anything is cached at all.** A point-prompted segmenter splits its work in +two: an encode that reads the whole image and costs most of the time, and a +decode from a click that costs almost none. D5 on #424 budgets =<300 ms for the +perceived cost of a click, and that number is only reachable if the first click +on an asset pays the encode and the ones after it do not. So the embedding is +kept, and this is what keeps it. + +**Bounded by count rather than by bytes, and small.** A byte budget would need to +know the size of a tensor living on a device this module must not import, and +would then be guessing at how much of that device somebody else's model is +holding. A count is honest about being a policy rather than a measurement, and +the arithmetic behind each default below is written down so the next person can +redo it rather than re-derive it. + +**In-process, so it dies with the process, and that is correct.** A suggestion is +not a fact about the workspace — nothing here is persisted, invalidated or +shared between workers — it is a saved intermediate that a restart is free to +recompute. Anything durable would be infrastructure this slice deliberately does +not add. +""" + +from __future__ import annotations + +from collections import OrderedDict +from typing import Final + +DEFAULT_EMBEDDING_CAPACITY: Final = 8 +"""How many assets' image embeddings to keep. + +For a hiera-base-plus-shaped encoder at 1024x1024 the feature maps come to +roughly 8 MB in half precision, so eight of them is on the order of 64 MB of +whatever device the model is on — comfortably inside the margin D1 says +base-plus leaves on a 16 GB profile, and enough that moving back and forth +between a handful of assets stays warm. Somebody labelling one asset at a time +never reaches the bound at all. +""" + +DEFAULT_PROVIDER_CAPACITY: Final = 2 +"""How many loaded models to keep resident. + +Two, because that is the co-residency D1 describes: a segmenter answering clicks +and a detector answering words, both live, without a third quietly arriving to +push one out. Weights are gigabytes — this is the bound that matters — and the +alternative to keeping them is re-reading them per click, which is the same +latency failure the embedding cache exists to prevent, one level up. +""" + + +class BoundedCache[K, V]: + """Bounded, least-recently-used, and deliberately not thread safe. + + Not thread safe for the same reason ``LocalTransformersProvider`` is not: a + worker process runs one task at a time, and a server handler holding a + model is already serialised by the device it is talking to. A lock here + would buy nothing and would suggest a concurrency this design does not + have. + + ``get`` counts as a use, which is what makes this LRU rather than + first-in-first-out: the asset somebody is clicking on repeatedly is the one + that must survive, and it is the one that would be evicted first under + insertion order. + """ + + def __init__(self, capacity: int) -> None: + if capacity < 1: + raise ValueError(f"a cache holding {capacity} things is not a cache") + self._capacity = capacity + self._held: OrderedDict[K, V] = OrderedDict() + + @property + def capacity(self) -> int: + """The most this will ever hold. Fixed for the life of the cache.""" + return self._capacity + + def get(self, key: K) -> V | None: + """What is held under that key, or ``None`` — and a hit is a use.""" + if key not in self._held: + return None + self._held.move_to_end(key) + return self._held[key] + + def put(self, key: K, value: V) -> V: + """Hold that, evicting the least recently used if the bound is reached. + + Returns the value, so a caller can write ``return cache.put(k, compute())`` + rather than putting and then reading back. + """ + if key in self._held: + self._held.move_to_end(key) + self._held[key] = value + while len(self._held) > self._capacity: + self._held.popitem(last=False) + return value + + def discard(self, key: K) -> None: + """Forget that key if it is held. A no-op if it is not.""" + self._held.pop(key, None) + + def clear(self) -> None: + """Forget everything.""" + self._held.clear() + + def __len__(self) -> int: + return len(self._held) + + def __contains__(self, key: object) -> bool: + """Membership **without** counting as a use, for tests that assert eviction.""" + return key in self._held diff --git a/src/visionset/inference/masks.py b/src/visionset/inference/masks.py new file mode 100644 index 00000000..31b42f79 --- /dev/null +++ b/src/visionset/inference/masks.py @@ -0,0 +1,285 @@ +# usage: from visionset.inference.masks import geometry_from +"""A binary mask in, one domain geometry out — and nothing about torch in between. + +A segmenter answers with a grid of booleans; this domain stores boxes and +polygons. That conversion is the whole of this module, and it is written over +plain Python sequences rather than tensors for the reason ``nms`` is: it is the +part of a segmentation adapter that can be wrong in a way no GPU is needed to +see, so it is the part a test drives with literals. + +**Which shape is produced is the caller's schema decision, not this module's +guess.** ``geometry_from`` takes the geometry kinds the active class actually +admits and produces one of those or nothing — D3 on #424, where a class allowing +polygons gets the outline, a class allowing only boxes gets the mask's extent, +and a class allowing neither is not offered the gesture at all. + +**Tolerance is relative, and that is what makes one "detail" setting work.** D3 +asks for a single knob that lands typical objects in a 10-40 vertex range. An +absolute pixel tolerance cannot: three pixels is nothing on a car and is the +whole of a bottle cap. So the tolerance handed to Douglas-Peucker is a fraction +of the region's own bounding diagonal, which makes the vertex count a property of +the *shape* rather than of how much of the frame it happens to fill. +""" + +from __future__ import annotations + +from collections.abc import Sequence +from typing import Final + +from visionset.kernel.domain import BboxGeometry, Geometry, GeometryType, PolygonGeometry + +Mask = Sequence[Sequence[bool]] +"""Rows of columns, ``mask[y][x]`` — the orientation every image library agrees +on and the one ``post_process_masks`` produces.""" + +Point = tuple[float, float] + +DEFAULT_DETAIL: Final = 0.01 +"""D3's single "detail" setting, as a fraction of the region's bounding diagonal. + +Chosen against the shape the range was written for rather than by taste: for a +roughly circular object this keeps the vertices where the sagitta of a chord +exceeds the tolerance, which works out at ~13 vertices — inside D3's 10-40 band +with room on both sides for shapes more and less convoluted than a circle. +Smaller means more faithful and more vertices; larger means fewer. +""" + +MINIMUM_TOLERANCE: Final = 0.5 +"""No tolerance below half a pixel, however small the region. + +Below this the simplification is arguing about detail the mask does not have — +its own coordinates are integers — and the vertex count runs away for nothing. +""" + + +def spans(mask: Mask) -> list[tuple[int, int, int]]: + """``(y, first_x, last_x)`` for every row holding anything. + + Rows rather than pixels on purpose. A full-resolution mask is a megapixel of + booleans, and anything here that looped over it in Python would cost more + than the forward pass that produced it; ``list.index`` does the scan in C and + the loop only ever sees the number of rows. + """ + found: list[tuple[int, int, int]] = [] + for y, row in enumerate(mask): + try: + first = row.index(True) # type: ignore[attr-defined] + except ValueError: + continue + last = len(row) - 1 - list(reversed(row)).index(True) + found.append((y, first, last)) + return found + + +def bbox_from(mask: Mask) -> BboxGeometry | None: + """The mask's extent, or ``None`` if it holds nothing. + + ``None`` rather than a raise: an empty mask is an ordinary answer from a + model asked about an empty patch of sky, and the caller turns it into "no + suggestion" rather than into an error. + + The box is the pixels' outer edge, so a single lit pixel is one wide and one + tall rather than zero — the domain refuses a zero-area box, and a + one-pixel-wide object is a real thing to point at. + """ + rows = spans(mask) + if not rows: + return None + y_min = rows[0][0] + y_max = rows[-1][0] + x_min = min(first for _, first, _ in rows) + x_max = max(last for _, _, last in rows) + return BboxGeometry( + x=float(x_min), + y=float(y_min), + width=float(x_max - x_min + 1), + height=float(y_max - y_min + 1), + ) + + +def outline(mask: Mask) -> list[Point]: + """The boundary of the blob the topmost-leftmost lit pixel belongs to. + + Moore-neighbourhood tracing with Jacob's stopping criterion: walk the ring of + lit pixels, at each one resuming the search from where the previous step + arrived, and stop on re-entering the start pixel from the direction first + used to leave it. Stopping merely on *reaching* the start again is the + classic bug — a shape with a one-pixel isthmus revisits its start mid-trace + and the outline comes back truncated. + + **One blob, not all of them.** A component-labelling pass would be a second + walk over every pixel to answer a question this caller does not have: a + point-prompted segmenter answers with the thing under the point, and the + speckle that survives its own post-processing is not a second object worth + tracing. A mask holding two real blobs yields the first one found. + """ + rows = spans(mask) + if not rows: + return [] + start = (rows[0][1], rows[0][0]) + height, width = len(mask), len(mask[0]) + + def lit(point: tuple[int, int]) -> bool: + x, y = point + return 0 <= x < width and 0 <= y < height and bool(mask[y][x]) + + # Clockwise from due west, which is where a scan arriving from the left came + # from — so the first candidate examined is the one just above the start. + around: Final = ((-1, 0), (-1, -1), (0, -1), (1, -1), (1, 0), (1, 1), (0, 1), (-1, 1)) + + traced = [start] + current, entered_from = start, 0 + while True: + for step in range(1, len(around) + 1): + index = (entered_from + step) % len(around) + candidate = (current[0] + around[index][0], current[1] + around[index][1]) + if lit(candidate): + # The direction the *next* search resumes from: back the way we + # came, which is the opposite neighbour. + entered_from = (index + len(around) // 2) % len(around) + current = candidate + break + else: + # An isolated pixel has no ring to walk. + return [(float(start[0]), float(start[1]))] + if current == start: + break + traced.append(current) + if len(traced) > 4 * (height + width): + # A boundary longer than any real one is a trace that failed to + # close. Returning what was walked beats looping. + break + return [(float(x), float(y)) for x, y in traced] + + +def _distance_to_segment(point: Point, start: Point, end: Point) -> float: + """Perpendicular distance, degenerating to plain distance on a zero-length segment.""" + (px, py), (sx, sy), (ex, ey) = point, start, end + dx, dy = ex - sx, ey - sy + if dx == 0.0 and dy == 0.0: + return ((px - sx) ** 2 + (py - sy) ** 2) ** 0.5 + return abs(dy * px - dx * py + ex * sy - ey * sx) / ((dx * dx + dy * dy) ** 0.5) + + +def simplified(points: Sequence[Point], *, tolerance: float) -> list[Point]: + """Douglas-Peucker over an open polyline. + + Iterative rather than recursive: a traced boundary is thousands of points + long and the recursive spelling of this algorithm is depth-unbounded on + exactly the input it is given here. + """ + if len(points) < 3: + return list(points) + keep = [False] * len(points) + keep[0] = keep[-1] = True + pending = [(0, len(points) - 1)] + while pending: + first, last = pending.pop() + if last <= first + 1: + continue + worst, distance = first, -1.0 + for index in range(first + 1, last): + found = _distance_to_segment(points[index], points[first], points[last]) + if found > distance: + worst, distance = index, found + if distance > tolerance: + keep[worst] = True + pending.append((first, worst)) + pending.append((worst, last)) + return [point for point, kept in zip(points, keep, strict=True) if kept] + + +def tolerance_for(points: Sequence[Point], *, detail: float) -> float: + """The pixel tolerance ``detail`` means for a region of this size. + + See the module docstring: a fraction of the bounding diagonal, floored so it + never argues about sub-pixel detail. + """ + if not points: + return MINIMUM_TOLERANCE + xs = [x for x, _ in points] + ys = [y for _, y in points] + diagonal = ((max(xs) - min(xs)) ** 2 + (max(ys) - min(ys)) ** 2) ** 0.5 + return max(MINIMUM_TOLERANCE, detail * diagonal) + + +def polygon_from(mask: Mask, *, detail: float = DEFAULT_DETAIL) -> PolygonGeometry | None: + """The mask's outline, simplified — or ``None`` if no polygon can be made. + + ``None`` covers both an empty mask and a blob too thin to have three distinct + corners: the domain requires three points, and a two-point "polygon" is a + line somebody would have to fix by hand rather than a suggestion worth + offering. + """ + traced = outline(mask) + if len(traced) < 3: + return None + tolerance = tolerance_for(traced, detail=detail) + kept = _closed(simplified(traced, tolerance=tolerance), tolerance=tolerance) + if len(kept) < 3: + return None + return PolygonGeometry(points=kept) + + +def _closed(kept: list[Point], *, tolerance: float) -> list[Point]: + """Drop the vertices Douglas-Peucker only kept because it was told to. + + The algorithm pins the first and last point of what it is given, and what it + is given here is a *ring* cut open at an arbitrary pixel. So the final vertex + is pinned for a reason that stops being true the moment the ring closes, and + it lands one pixel from the first — a stray handle on an otherwise clean + outline, most visible on the straight-edged shapes where it is least + excusable: an axis-aligned rectangle came back as five points. + + Judged by the same tolerance as everything else rather than by exact + equality: the artifact is a near-duplicate, not a duplicate, so testing + ``kept[0] == kept[-1]`` never fires on the case that motivates it. + """ + while len(kept) > 3: + if _distance_to_segment(kept[-1], kept[-2], kept[0]) > tolerance: + return kept + kept = kept[:-1] + return kept + + +def bounds_of(points: Sequence[Point]) -> BboxGeometry | None: + """The smallest box containing those points, or ``None`` if there are none. + + Zero-area is widened to one unit rather than refused, for the reason + :func:`bbox_from` gives about a single lit pixel: the domain will not store a + degenerate box, and a perfectly flat or vertical outline is a real thing for + a segmenter to find at the edge of an image. + """ + if not points: + return None + xs = [x for x, _ in points] + ys = [y for _, y in points] + return BboxGeometry( + x=min(xs), + y=min(ys), + width=max(max(xs) - min(xs), 1.0), + height=max(max(ys) - min(ys), 1.0), + ) + + +def narrowed(geometry: Geometry, *, allowed: Sequence[GeometryType]) -> Geometry | None: + """That shape in a kind the active class admits, or ``None`` if there is none. + + D3's rule, and it lives here — above the adapter and below the route — + because it is a *schema* decision rather than a model one. The port carries + no notion of what a class allows, deliberately: widening it would push a + project's schema into a protocol that has to be implementable by a service + that has never heard of this workspace. + + So a segmenter answers with the most informative shape it has, and the + narrowing happens here: a polygon stands where polygons are allowed, becomes + its own bounding box where only boxes are, and is refused where neither is — + the tag-only class D3 says the gesture is not offered for at all. Nothing is + ever widened, because a box cannot become the outline it never held. + """ + kinds = set(allowed) + if geometry.type in kinds: + return geometry + if isinstance(geometry, PolygonGeometry) and GeometryType.BBOX in kinds: + return bounds_of(geometry.points) + return None diff --git a/src/visionset/inference/providers.py b/src/visionset/inference/providers.py new file mode 100644 index 00000000..be2ca99e --- /dev/null +++ b/src/visionset/inference/providers.py @@ -0,0 +1,196 @@ +# usage: from visionset.inference import provider_for, resident +"""Which adapter answers for a connection, and keeping the loaded one around. + +Two questions that belong together because the answer to the second depends on +the first being cheap to ask. + +**Resolution is by the model's own declared family, not by the connection's +kind.** ``ConnectionType`` says *where* a model runs — here or elsewhere — and +that is the only thing it says. It cannot say whether the weights behind a local +connection are a detector or a segmenter, and those answer different questions: +one takes words and one takes places. So the family is read from the model's own +config, which is a small JSON file already sitting in the cache beside the +weights the connection downloaded. A connection pointed at a detector and asked +with points is then refused with the port's own vocabulary rather than dying +somewhere inside a forward pass on a shape mismatch. + +**Loaded models are kept, because the alternative defeats the embedding cache.** +D5 on #424 budgets =<300 ms for a click. A provider built fresh per request would +re-read gigabytes of weights every time and would carry an empty embedding cache +into every click — so the per-asset encode would happen on every click too, and +the two caches would each be defeated by the absence of the other. Keeping the +provider is what makes keeping the embedding worth anything. + +**Keyed on the connection's identity *and* its last edit**, so changing the model +id, the device or the precision builds a new provider rather than silently +serving the old weights under new settings. Deleting a connection needs no +special handling: nothing will ask for that key again, and the bound evicts it. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any, Final + +from visionset.inference._extra import imported, require +from visionset.inference.cache import DEFAULT_PROVIDER_CAPACITY, BoundedCache +from visionset.inference.sam_provider import LocalSamProvider +from visionset.inference.transformers_provider import LocalTransformersProvider +from visionset.inference.weights import cache_root +from visionset.kernel.domain import ConnectionSetupState, ConnectionType, InferenceConnection +from visionset.kernel.errors import ( + InferenceConnectionNotRunnable, + InferenceConnectionNotSetUp, +) +from visionset.kernel.ports import ModelProvider + +SEGMENTER_FAMILIES: Final[frozenset[str]] = frozenset({"sam2"}) +"""``model_type`` values this build serves with the point-prompted adapter. + +A set rather than a single string because the family is what matters and its +members grow: the video variant of the same architecture is the 0.2.0 door D1 +keeps open, and it arrives here as one more name rather than as a second +resolution mechanism. Anything not named here is served by the detector adapter, +which is the older and more common case. +""" + +_Key = tuple[str, str] + + +class ProviderPool: + """Loaded providers, bounded and least-recently-used. + + Instantiable rather than only module-level so a test can hold its own and + assert on it without reaching into process state — and so two workspaces in + one process cannot end up sharing a provider through a global. + """ + + def __init__(self, capacity: int = DEFAULT_PROVIDER_CAPACITY) -> None: + self._held: BoundedCache[_Key, ModelProvider] = BoundedCache(capacity) + self._builds = 0 + + @property + def builds(self) -> int: + """How many providers this pool has actually constructed. + + The counter that separates a working pool from one that rebuilds every + time — both answer correctly, and only this tells them apart. + """ + return self._builds + + def get(self, connection: InferenceConnection, *, workspace_root: Path) -> ModelProvider: + """The provider for that connection, built once and kept. + + Every refusal ``provider_for`` can raise is raised here too, and raised + *before* anything is cached: a connection that is not ready must not + leave a half-answer behind for the request that follows its download. + """ + key = (str(connection.id), connection.updated_at.isoformat()) + held = self._held.get(key) + if held is not None: + return held + built = provider_for(connection, workspace_root=workspace_root) + self._builds += 1 + return self._held.put(key, built) + + def clear(self) -> None: + """Drop everything held. What a test does between cases.""" + self._held.clear() + + def __len__(self) -> int: + return len(self._held) + + +_RESIDENT: Final = ProviderPool() + + +def resident() -> ProviderPool: + """The process-wide pool. + + A function rather than the object itself so that importing this module does + not read as taking a handle on shared state, and so the one place it is + reached from is greppable. + """ + return _RESIDENT + + +def provider_for(connection: InferenceConnection, *, workspace_root: Path) -> ModelProvider: + """The thing that will answer for this connection, or the reason nothing can. + + Builds a provider without loading any weights — the load is lazy, in the + adapter — so a caller may construct one to find out whether it *could* run. + That is what makes the refusals here worth raising early. + + Raises: + InferenceConnectionNotSetUp: a local connection whose weights are not + here yet. The message names ``download_weights``, because that is the + action that makes the identical call succeed. + InferenceConnectionNotRunnable: nothing in this build runs a connection + of that kind. + LocalInferenceUnavailable: the optional runtime is not installed. + """ + match connection.connection_type: + case ConnectionType.LOCAL: + return _local(connection, workspace_root=workspace_root) + case ConnectionType.HTTP: + raise InferenceConnectionNotRunnable( + f"connection {connection.name!r} is an http connection, and this build has no " + "adapter that can speak to one; use a local connection, or a later version" + ) + + +def _local(connection: InferenceConnection, *, workspace_root: Path) -> ModelProvider: + """A local provider of whichever family this connection's model belongs to. + + The order of the two checks is deliberate and unchanged from the slice that + introduced it: the connection's own state first, because "your weights are + not here" is about something the caller can fix from where they are standing, + while a missing extra is about the installation and is the same answer for + every connection in the workspace. + """ + if connection.setup_state is not ConnectionSetupState.READY: + raise InferenceConnectionNotSetUp( + f"connection {connection.name!r} has no weights on this machine yet; " + "run its download_weights action first" + ) + require() + # ``device`` is non-null on a local connection — the domain's cross-field + # rule is what makes that true — so this narrows for the type checker rather + # than handling a possibility. + assert connection.device is not None + cache_dir = cache_root(workspace_root) + common: dict[str, Any] = { + "device": connection.device, + "precision": connection.precision, + "cache_dir": cache_dir, + "connection_name": connection.name, + } + if family_of(connection, cache_dir=cache_dir) in SEGMENTER_FAMILIES: + return LocalSamProvider(connection.model_id, connection.model_revision, **common) + return LocalTransformersProvider(connection.model_id, connection.model_revision, **common) + + +def family_of(connection: InferenceConnection, *, cache_dir: Path) -> str: + """The ``model_type`` the downloaded config declares, or ``""`` if it cannot say. + + Read from the cache rather than from the network — ``local_files_only`` — for + the same reason every other load in this package is: this product downloads + weights when somebody asks it to and at no other time. + + An unreadable or unrecognised config answers ``""`` rather than raising, and + ``""`` resolves to the detector. A connection whose config cannot be parsed + is going to fail at load time with the library's own message, which says far + more about what is wrong with those files than anything this function could + invent from having failed to read one field. + """ + transformers = imported("transformers") + try: + config = transformers.AutoConfig.from_pretrained( + connection.model_id, + revision=connection.model_revision, + cache_dir=str(cache_dir), + local_files_only=True, + ) + except Exception: # noqa: BLE001 — see the docstring: this is a fallback, not a handler + return "" + return str(getattr(config, "model_type", "") or "") diff --git a/src/visionset/inference/sam_provider.py b/src/visionset/inference/sam_provider.py new file mode 100644 index 00000000..99f0a915 --- /dev/null +++ b/src/visionset/inference/sam_provider.py @@ -0,0 +1,305 @@ +# usage: from visionset.inference import LocalSamProvider +"""A ``ModelProvider`` that answers a pointing gesture, here, in this process. + +The counterpart to ``transformers_provider``: that one answers words and refuses +points, this one answers points and refuses words. Neither guesses at the other, +because a connection names one model and a model of this family genuinely cannot +do the other job. + +**The whole design is one split.** A segmenter of this family reads the image +once into an embedding — the expensive half — and then answers any number of +clicks from that embedding almost for free. D5 on #424 budgets =<300 ms for the +perceived cost of a click, and that number is only reachable if refining a +suggestion never re-reads the image. ``transformers`` draws the same line the +design does: :meth:`get_image_embeddings` is the encode, and the processor +accepts ``original_sizes`` *without* ``images``, so the decode never touches a +pixel. The cache sits exactly on that seam. + +**Nothing about the cache is visible through the port.** ``ModelProvider`` must +stay implementable by something running in another building (the recorded +decision on #418), so the caching is an adapter's private business: the protocol +gets ``predict``, and a hosted segmenter is free to cache in whatever way its +own deployment allows, or not at all. + +The fp16 shims and the missing-extra error are ``transformers_provider``'s, +reused rather than respelled — same ``_fp16.forward_guard``, same +``_extra.imported``. +""" + +from __future__ import annotations + +import logging +from collections.abc import Iterator +from io import BytesIO +from pathlib import Path +from typing import Any, Final +from uuid import UUID + +from PIL import Image + +from visionset.inference import _fp16 +from visionset.inference._extra import imported +from visionset.inference.cache import DEFAULT_EMBEDDING_CAPACITY, BoundedCache +from visionset.inference.masks import DEFAULT_DETAIL, polygon_from +from visionset.kernel.domain import ( + AssetPrediction, + PointPrompt, + PredictedRegion, + PredictionRequest, + PredictionTarget, +) +from visionset.kernel.errors import UnsupportedPrompt + +_logger: Final = logging.getLogger(__name__) + +POSITIVE: Final = 1 +NEGATIVE: Final = 0 +"""What this family calls a point that says *this* and one that says *not that*. + +The domain spells the same distinction as two tuples on +:class:`~visionset.kernel.domain.PointPrompt`, which is how a person thinks about +it. These two integers are how the model does, and the translation between them +is exactly the kind of thing that lives in an adapter. +""" + +CPU_FALLBACK_WARNING: Final = ( + "inference connection %r asks for device %r, which this machine does not offer; " + "running on the CPU in full precision instead" +) + + +def points_and_labels(prompt: PointPrompt) -> tuple[list[list[float]], list[int]]: + """The prompt as this family wants it: one flat point list, one label list. + + Pure, and separated from the forward for the same reason ``regions_from`` is + in the detector adapter — positive and negative points arriving in the wrong + order, or with the labels off by one, is a bug that needs no GPU to catch and + would otherwise need one to see. + + Order is positives then negatives. Nothing in the model depends on it, and + fixing it makes the conversion testable by equality rather than by set + comparison. + """ + points = [[float(x), float(y)] for x, y in prompt.positive] + labels = [POSITIVE] * len(points) + points += [[float(x), float(y)] for x, y in prompt.negative] + labels += [NEGATIVE] * (len(points) - len(labels)) + return points, labels + + +def best_of(iou_scores: list[float], masks: list[Any]) -> tuple[int, float]: + """Which of the multi-mask answers to offer, and how sure it is. + + A segmenter of this family answers a single click with several masks at + different scales — the object, the part, the whole — because a click is + ambiguous about which was meant. Offering all three would make the user + disambiguate a thing they did not ask about; offering the highest-scoring one + is what the model's own IoU head is for, and refining with a second click is + how the design says to resolve the ambiguity instead (D2). + """ + if not iou_scores: + return 0, 0.0 + best = max(range(len(iou_scores)), key=lambda index: iou_scores[index]) + return best, min(1.0, max(0.0, float(iou_scores[best]))) + + +class LocalSamProvider: + """Runs a point-promptable segmenter here, on this machine. + + Satisfies :class:`~visionset.kernel.ports.ModelProvider` structurally, like + its sibling, and is built by the composition root that has already decided + this connection's model is of this family. + + One instance per connection, held across requests by the provider cache — + which is what makes the embedding cache inside it worth anything. A provider + built fresh per click would have an empty cache every time and would pay the + encode on every click, which is the exact latency failure D5 names. + """ + + def __init__( + self, + model_id: str, + model_revision: str, + *, + device: str, + precision: str | None, + cache_dir: Path, + connection_name: str = "", + detail: float = DEFAULT_DETAIL, + embedding_capacity: int = DEFAULT_EMBEDDING_CAPACITY, + ) -> None: + self._model_id = model_id + self._model_revision = model_revision + self._device = device + self._precision = precision + self._cache_dir = cache_dir + self._connection_name = connection_name + self._detail = detail + self._loaded: tuple[Any, Any, str, bool] | None = None + self._embeddings: BoundedCache[UUID, tuple[Any, tuple[int, int]]] = BoundedCache( + embedding_capacity + ) + self._encodes = 0 + + @property + def model_ref(self) -> str: + """``id@revision`` — the string an accepted annotation will carry.""" + return f"{self._model_id}@{self._model_revision}" + + @property + def encodes(self) -> int: + """How many times an image has actually been read into an embedding. + + Exposed because it is the only externally visible difference between a + cache that works and one that does not: both answer correctly, and only + this counter separates one click's worth of work from two. The test that + proves D5's encode-once behaviour reads it, and would pass on a bypassed + cache without it. + """ + return self._encodes + + def predict(self, request: PredictionRequest) -> Iterator[AssetPrediction]: + """One answer per target, yielded as each finishes. + + Raises: + UnsupportedPrompt: the request asks with words, and this is a + segmenter. It answers places. + LocalInferenceUnavailable: the optional runtime is not installed. + """ + if not isinstance(request.prompt, PointPrompt): + raise UnsupportedPrompt( + f"{self.model_ref} answers point prompts; it was asked with " + f"{request.prompt.kind!r}, which it has no way to interpret" + ) + points, labels = points_and_labels(request.prompt) + torch = imported("torch") + for target in request.targets: + yield self._one( + target, + points=points, + labels=labels, + torch=torch, + minimum_confidence=request.minimum_confidence, + ) + + def _one( + self, + target: PredictionTarget, + *, + points: list[list[float]], + labels: list[int], + torch: Any, + minimum_confidence: float, + ) -> AssetPrediction: + """One image, one prompt, one region at most.""" + processor, model, device, half = self._ready() + embedding, size = self._embedding(target, processor=processor, model=model, device=device) + height, width = size + inputs = processor( + original_sizes=[[height, width]], + input_points=[[points]], + input_labels=[[labels]], + return_tensors="pt", + ).to(device) + with _fp16.forward_guard(torch, device_type=device.split(":")[0], half=half): + outputs = model( + input_points=inputs["input_points"], + input_labels=inputs["input_labels"], + image_embeddings=embedding, + multimask_output=True, + ) + return AssetPrediction( + asset_id=target.asset_id, + model_ref=self.model_ref, + regions=self._regions( + outputs, + processor=processor, + size=size, + minimum_confidence=minimum_confidence, + ), + ) + + def _regions( + self, + outputs: Any, + *, + processor: Any, + size: tuple[int, int], + minimum_confidence: float, + ) -> tuple[PredictedRegion, ...]: + """The chosen mask as a domain polygon, or nothing at all. + + Empty rather than raising, in all three of the ways this can come back + with no answer — the model was not sure enough, the mask was empty, or + the blob was too thin to be a polygon. A click on a patch of sky is an + ordinary thing to do and "no suggestion" is the honest reply to it. + + **The label is deliberately empty.** Pointing says *where*, not *what*: + this model has no vocabulary and answers with a shape. The editor already + knows which class is active — that is what chose the geometry kinds — so + a name invented here would be a second, worse source for something the + caller already holds. + """ + lifted = processor.post_process_masks( + outputs.pred_masks, original_sizes=[list(size)], binarize=True + )[0] + scores = [float(value) for value in outputs.iou_scores.flatten().tolist()] + chosen, confidence = best_of(scores, lifted) + if confidence < minimum_confidence: + return () + mask = lifted.reshape(-1, *lifted.shape[-2:])[chosen] + polygon = polygon_from(mask.tolist(), detail=self._detail) + if polygon is None: + return () + return (PredictedRegion(label="", confidence=confidence, geometry=polygon),) + + def _embedding( + self, target: PredictionTarget, *, processor: Any, model: Any, device: str + ) -> tuple[Any, tuple[int, int]]: + """This asset's image embedding, computed once and kept. + + Keyed on ``asset_id`` alone, which is sound because assets are + content-addressed: the bytes behind an id cannot change, so a hit can + never be stale. Editing the *connection* is what would invalidate these, + and that replaces the whole provider rather than reaching in here. + """ + held = self._embeddings.get(target.asset_id) + if held is not None: + return held + image = Image.open(BytesIO(target.content)).convert("RGB") + size = (image.height, image.width) + inputs = processor(images=image, return_tensors="pt").to(device) + self._encodes += 1 + return self._embeddings.put( + target.asset_id, (model.get_image_embeddings(inputs["pixel_values"]), size) + ) + + def _ready(self) -> tuple[Any, Any, str, bool]: + if self._loaded is None: + self._loaded = self._load() + return self._loaded + + def _load(self) -> tuple[Any, Any, str, bool]: + torch = imported("torch") + transformers = imported("transformers") + device, half = self._resolved_device(torch) + common = { + "revision": self._model_revision, + "cache_dir": str(self._cache_dir), + "local_files_only": True, + } + processor = transformers.AutoProcessor.from_pretrained(self._model_id, **common) + model = transformers.Sam2Model.from_pretrained( + self._model_id, + dtype=torch.float16 if half else torch.float32, + **common, + ) + return processor, model.to(device).eval(), device, half + + def _resolved_device(self, torch: Any) -> tuple[str, bool]: + """Where this runs, and whether half precision survives — the sibling's rule.""" + wanted = self._device.strip() + if wanted.startswith("cuda") and not torch.cuda.is_available(): + _logger.warning(CPU_FALLBACK_WARNING, self._connection_name, wanted) + return "cpu", False + return wanted, wanted.startswith("cuda") and _fp16.wants_half(self._precision) diff --git a/src/visionset/inference/suggestions.py b/src/visionset/inference/suggestions.py new file mode 100644 index 00000000..b5658893 --- /dev/null +++ b/src/visionset/inference/suggestions.py @@ -0,0 +1,121 @@ +# usage: from visionset.inference import suggest +"""One click, one suggestion — the orchestration behind the editor's gesture. + +**Here rather than in a route, because every surface would need the same thing.** +A route, a command and a tool would each have to resolve a connection, read an +asset's bytes, run a provider and narrow the answer to what the active class +admits; that is four steps of policy, and policy shared by surfaces moves down. +It cannot move all the way down into ``visionset.kernel`` — running a model means +torch — so it lives here, beside the adapters, exactly as ``fetch_weights`` does. + +**Nothing is written.** A suggestion is a proposal: this returns it and forgets +it, and the annotation it may become is created later through the ordinary write +path, by a caller that carries the ``provenance``, ``model_ref`` and +``confidence`` an accepted suggestion earns (D4 on #424). The only thing that +outlives the call is the cached embedding, which is an optimisation and not a +record. + +**One asset, one prompt set.** Batch invocation is #425's, and the shape here +says so: a single target in, a single answer out. +""" + +from __future__ import annotations + +from uuid import UUID + +from visionset.inference.masks import DEFAULT_DETAIL, narrowed +from visionset.inference.providers import ProviderPool, resident +from visionset.kernel.domain import ( + AssetPrediction, + GeometryType, + PointPrompt, + PredictedRegion, + PredictionRequest, + PredictionTarget, + media_type_of, +) +from visionset.kernel.services import ( + InferenceConnectionService, + IngestService, + WorkspaceService, +) + + +def suggest( + workspace: WorkspaceService, + *, + project_id: UUID, + asset_id: UUID, + connection_id: UUID, + prompt: PointPrompt, + allowed: tuple[GeometryType, ...], + detail: float = DEFAULT_DETAIL, + minimum_confidence: float = 0.0, + pool: ProviderPool | None = None, +) -> AssetPrediction: + """What the model proposes for that click, in a shape that class can hold. + + The order of the two lookups is the order of the refusals a caller most + needs. The connection is resolved first because "no weights here yet" and + "this build cannot run that kind" are answers about the *setup* somebody is + part-way through, and getting them before an asset lookup means a caller + fixing their configuration is not also told their asset is fine. + + An empty ``regions`` is a real answer and not a failure: the model was asked + about a patch of sky, or was not sure enough, or the shape it found cannot be + expressed in the kinds this class admits. Every one of those is "no + suggestion", and none of them is an error somebody made. + + Raises: + InferenceConnectionNotFound: no such connection in this workspace. + InferenceConnectionNotSetUp: a local connection whose weights are not here. + InferenceConnectionNotRunnable: nothing in this build runs that kind. + LocalInferenceUnavailable: the optional runtime is not installed. + UnsupportedPrompt: that connection's model answers words, not places. + ProjectNotFound: no such project. + AssetNotFound: no such asset in that project. + """ + connection = InferenceConnectionService(workspace).get(connection_id) + provider = (pool or resident()).get(connection, workspace_root=workspace.root) + + ingest = IngestService(workspace) + asset = ingest.asset(project_id, asset_id) + with ingest.open_content(asset) as handle: + content = handle.read() + + request = PredictionRequest( + targets=( + PredictionTarget( + asset_id=asset.id, content=content, media_type=media_type_of(asset.format) + ), + ), + prompt=prompt, + minimum_confidence=minimum_confidence, + ) + # ``predict`` yields, and this slice asks about exactly one asset — so one + # ``next`` is the whole of the answer. A provider that yielded nothing at all + # would be breaking the port's contract rather than reporting no findings, + # which is what the default guards against. + prediction = next( + iter(provider.predict(request)), + AssetPrediction(asset_id=asset.id, model_ref="", regions=()), + ) + return prediction.model_copy(update={"regions": _in_kinds(prediction.regions, allowed)}) + + +def _in_kinds( + regions: tuple[PredictedRegion, ...], allowed: tuple[GeometryType, ...] +) -> tuple[PredictedRegion, ...]: + """Every region the active class can actually hold, D3's rule applied. + + A region whose shape cannot be narrowed is dropped rather than offered in a + kind the schema would refuse: the write that followed would fail validation, + and a suggestion the product knows cannot be accepted is worse than no + suggestion at all. + """ + kept = [] + for region in regions: + geometry = narrowed(region.geometry, allowed=allowed) + if geometry is not None: + kept.append(region.model_copy(update={"geometry": geometry})) + return tuple(kept) diff --git a/src/visionset/kernel/domain/__init__.py b/src/visionset/kernel/domain/__init__.py index fdb74b67..26eb60c7 100644 --- a/src/visionset/kernel/domain/__init__.py +++ b/src/visionset/kernel/domain/__init__.py @@ -93,10 +93,13 @@ JobPayload, ) from visionset.kernel.domain.media import ( + MEDIA_TYPES, + OCTET_STREAM, ImageFormat, ImageMetadata, VideoFrame, VideoMetadata, + media_type_of, ) from visionset.kernel.domain.names import normalize_name from visionset.kernel.domain.partition import ( @@ -208,6 +211,8 @@ "INGEST_TRANSITIONS", "JOB_TRANSITIONS", "MANIFEST_VERSION", + "MEDIA_TYPES", + "OCTET_STREAM", "OPEN_JOB_STATES", "PROMOTABLE_PROGRESS", "PROMOTABLE_STATES", @@ -311,6 +316,7 @@ "generate_secret", "hash_secret", "initial_progress", + "media_type_of", "normalize_name", "partition_assets", "progress_after_annotating", diff --git a/src/visionset/kernel/domain/media.py b/src/visionset/kernel/domain/media.py index 7aafbf25..dcc7a478 100644 --- a/src/visionset/kernel/domain/media.py +++ b/src/visionset/kernel/domain/media.py @@ -37,7 +37,9 @@ from __future__ import annotations +from collections.abc import Mapping from enum import StrEnum +from typing import Final from pydantic import BaseModel, ConfigDict, Field @@ -56,6 +58,36 @@ class ImageFormat(StrEnum): PNG = "png" +MEDIA_TYPES: Final[Mapping[ImageFormat, str]] = { + ImageFormat.JPEG: "image/jpeg", + ImageFormat.PNG: "image/png", +} +"""The IANA name for each accepted encoding. + +Total over :class:`ImageFormat` and asserted so by a test, for the reason the +enum's own docstring gives about half-done extensions: a member added here +without a media type would otherwise degrade every download of it to +``octet-stream`` quietly, which is the failure that looks like nothing. + +Here rather than beside the route that serves bytes because it is a fact about +the format, and it now has two readers — the asset download and the inference +adapters, which must tell a provider what it is being handed. A second copy of a +two-line map is how a product ends up serving ``image/png`` on one surface and +``application/octet-stream`` on another for the same asset. +""" + +OCTET_STREAM: Final = "application/octet-stream" +"""For an asset written before the ingest pipeline probed formats. + +Nothing can invent what nobody measured, and admitting that beats guessing. +""" + + +def media_type_of(image_format: ImageFormat | None) -> str: + """The IANA media type for that format, or ``octet-stream`` for an unprobed one.""" + return OCTET_STREAM if image_format is None else MEDIA_TYPES[image_format] + + class ImageMetadata(BaseModel): """What one still image turns out to be: how big it is, and how it is encoded. diff --git a/src/visionset/server/models.py b/src/visionset/server/models.py index bfee24f5..e3e6e15c 100644 --- a/src/visionset/server/models.py +++ b/src/visionset/server/models.py @@ -1677,3 +1677,71 @@ class ConnectionUpdate(BaseModel): device: str | None = None precision: str | None = None endpoint_url: str | None = None + + +class SuggestPoint(BaseModel): + """One click, in the asset's own pixel coordinates. + + An object rather than a two-element array because a JSON ``[x, y]`` is a + shape a generated client types as ``number[]`` and a reader has to guess the + order of. The domain's own tuples are fine — Python has positional meaning — + but the wire is read by people. + """ + + x: float + y: float + + +class SuggestRequest(BaseModel): + """Where somebody clicked, on what, through which connection. + + Everything travels in the body rather than in the path: the call names an + asset *and* a connection, and neither owns the other. Putting one in the path + would make it look like the parent of the request, which is how a URL is + read. + """ + + project_id: UUID + asset_id: UUID + connection_id: UUID + #: At least one point that says *this*. A gesture with only negative points + #: is not a refinement of anything. + positive: list[SuggestPoint] = Field(min_length=1) + #: Points that say *not that* — how somebody carves a hole out of an + #: over-eager first answer without starting the gesture over. + negative: list[SuggestPoint] = Field(default_factory=list) + #: The geometry kinds the active class admits. The server produces one of + #: these or nothing at all; it never answers in a kind the schema would go on + #: to refuse. Sent by the caller because the class is the caller's state — + #: the server would otherwise be guessing which class a click was meant for. + allowed_geometries: list[GeometryType] = Field(min_length=1) + #: How much detail to keep when an outline becomes a polygon, as a fraction + #: of the region's own size. Null takes the server's default, which is what + #: every ordinary caller sends. + detail: float | None = Field(default=None, gt=0.0, le=1.0) + + +class SuggestedRegion(BaseModel): + """One proposed shape and how sure the model is of it.""" + + geometry: Geometry + confidence: float = Field(ge=0.0, le=1.0) + + +class SuggestionOut(BaseModel): + """What the model proposes, or an honest nothing. + + ``region`` is null when there is no suggestion, and that is an ordinary + answer rather than an error: a click can land on sky, the model can be less + sure than the caller asked for, and the shape found can be one this class + cannot hold. A 404 or a 409 for any of those would be telling the caller they + did something wrong when they did not. + + ``model_ref`` is echoed on every answer, including the empty one, because it + is what an accepted suggestion has to carry into its annotation — and a + caller that had to remember which connection it asked would be keeping a + second copy of something the response can simply state. + """ + + model_ref: str + region: SuggestedRegion | None = None diff --git a/src/visionset/server/routes/__init__.py b/src/visionset/server/routes/__init__.py index fb540f39..d02e809f 100644 --- a/src/visionset/server/routes/__init__.py +++ b/src/visionset/server/routes/__init__.py @@ -66,6 +66,7 @@ # a stage of the data's life but a piece of this workspace's configuration, # which the pipeline reads rather than produces. inference.router, + inference.suggestions, # Last, and outside the pipeline order above on purpose: a background job is # not a stage of the data's life, it is how some of those stages run. Reading # it into the sequence would suggest a place it does not have. diff --git a/src/visionset/server/routes/assets.py b/src/visionset/server/routes/assets.py index c1f075cc..0e5b7e06 100644 --- a/src/visionset/server/routes/assets.py +++ b/src/visionset/server/routes/assets.py @@ -30,7 +30,7 @@ from fastapi.responses import StreamingResponse -from visionset.kernel.domain import Asset, ImageFormat +from visionset.kernel.domain import MEDIA_TYPES, OCTET_STREAM, Asset, media_type_of from visionset.kernel.ports import THUMBNAIL_FORMAT from visionset.kernel.services import ( BatchService, @@ -72,40 +72,23 @@ def _promoted(workspace: WorkspaceDep, project_id: UUID) -> frozenset[UUID]: return DatasetService(workspace).member_asset_ids(dataset.id) -#: What each ``ImageFormat`` is called on the wire. A mapping rather than -#: ``f"image/{format}"`` because the two coincide today and would stop coinciding -#: the moment a format whose media type is not its own name arrives — WEBP is -#: already named as the next member — and a wrong ``Content-Type`` is the kind of -#: bug that shows up in one browser and nowhere else. -#: -#: Indexed directly rather than with a fallback, the ``ProgressCounts`` bargain: -#: exhaustiveness is asserted by a test against the enum itself, so adding a -#: member without a media type fails the suite instead of quietly degrading every -#: download of it to ``octet-stream``. -_MEDIA_TYPES: Final[dict[ImageFormat, str]] = { - ImageFormat.JPEG: "image/jpeg", - ImageFormat.PNG: "image/png", -} - -#: For an asset written before the ingest pipeline probed formats. The store -#: cannot invent what nobody measured, and admitting that beats guessing. -_OCTET_STREAM: Final = "application/octet-stream" - # FastAPI documents a 200 as ``application/json`` unless told otherwise — the # app-level ``UNIVERSAL_ERROR_RESPONSES`` only covers 422/500/503 — so the binary # content type is declared per route. ``{}`` as the schema is OpenAPI's way of # saying "bytes, and there is nothing more to say about their shape". # -# Every type ``_media_type`` can return is listed, ``_OCTET_STREAM`` included. A +# Every type ``media_type_of`` can return is listed, ``OCTET_STREAM`` included. A # response the route really sends and the contract does not declare is a lie a # generated client inherits — and the pre-pipeline rows that produce it are # exactly the ones a caller is least prepared for. +# +# Built from ``MEDIA_TYPES`` rather than written out, so a format added to the +# domain cannot be served with a content type this contract never declared. _IMAGE_RESPONSE: Final[dict[int | str, dict[str, Any]]] = { 200: { "content": { - "image/jpeg": {"schema": {}}, - "image/png": {"schema": {}}, - _OCTET_STREAM: {"schema": {}}, + **{media_type: {"schema": {}} for media_type in sorted(MEDIA_TYPES.values())}, + OCTET_STREAM: {"schema": {}}, }, "description": "The bytes, streamed.", } @@ -120,7 +103,7 @@ def _promoted(workspace: WorkspaceDep, project_id: UUID) -> frozenset[UUID]: def _media_type(asset: Asset) -> str: - return _OCTET_STREAM if asset.format is None else _MEDIA_TYPES[asset.format] + return media_type_of(asset.format) @router.get("", responses=documented(404)) @@ -262,6 +245,6 @@ def get_asset_thumbnail( stream = ingest.open_thumbnail(asset) return StreamingResponse( stream, - media_type=_MEDIA_TYPES[THUMBNAIL_FORMAT], + media_type=MEDIA_TYPES[THUMBNAIL_FORMAT], headers={"ETag": f'"{asset.thumbnail_hash}"', "Cache-Control": _IMMUTABLE}, ) diff --git a/src/visionset/server/routes/inference.py b/src/visionset/server/routes/inference.py index 02687798..80e39211 100644 --- a/src/visionset/server/routes/inference.py +++ b/src/visionset/server/routes/inference.py @@ -22,10 +22,11 @@ from fastapi import Response, status +from visionset.inference import DEFAULT_DETAIL, suggest from visionset.inference import require as require_local_inference from visionset.jobs.weights import JOB_TYPE as download_job_type from visionset.jobs.weights import payload_for as download_payload_for -from visionset.kernel.domain import BackgroundJobSpec +from visionset.kernel.domain import BackgroundJobSpec, PointPrompt from visionset.kernel.services import InferenceConnectionService from visionset.server.dependencies import RunnerDep, WorkspaceDep, protected_router from visionset.server.errors import documented @@ -35,10 +36,20 @@ ConnectionOut, ConnectionPage, ConnectionUpdate, + SuggestedRegion, + SuggestionOut, + SuggestRequest, ) router = protected_router(prefix="/inference/connections", tags=["inference"]) +#: A second router because the path is a sibling of ``connections`` rather than a +#: child of one: a suggestion is made *through* a connection, not *on* it, and +#: nesting it under ``/inference/connections/{id}/suggest`` would put the asset — +#: the thing the call is actually about — in the body under a URL claiming the +#: connection owns it. +suggestions = protected_router(prefix="/inference", tags=["inference"]) + @router.get("") def list_inference_connections(workspace: WorkspaceDep) -> ConnectionPage: @@ -145,6 +156,61 @@ def download_connection_weights( return BackgroundJobOut.of(job) +@suggestions.post("/suggest", responses=documented(404, 409, 422)) +def suggest_region(workspace: WorkspaceDep, body: SuggestRequest) -> SuggestionOut: + """Propose a shape for the thing under those points. + + The server side of the editor's suggest gesture (`cf. #424`). One asset, one + prompt set, one answer — batch prediction is a separate path and is not this + one. + + **Nothing is written and nothing is remembered.** A suggestion is a proposal: + accepting it is a later, ordinary annotation write carrying `provenance: + model`, this response's `model_ref`, and its `confidence`. Discarding it + costs a request that already finished. The only thing that outlives the call + is a cached image embedding, which is an optimisation rather than a record — + so the same points sent twice answer the same way, and a restart changes + nothing but the latency of the first click. + + **The first click on an asset is the slow one.** A segmenter reads the whole + image once and then answers any number of clicks from that reading almost for + free, which is what makes refining by adding points practical. Sending the + accumulated points — rather than a diff — is what keeps this stateless. + + **`allowed_geometries` is the caller's schema, not a preference.** The answer + is produced in one of the kinds named or not at all: a class that admits + polygons gets the outline, a class that admits only boxes gets its extent, + and a class that admits neither gets `region: null`. Answering in a kind the + schema would refuse would produce a suggestion that cannot be accepted. + + A null `region` is a successful answer with nothing to propose. Refusals are + reserved for things the caller can act on: an unknown project, asset or + connection is 404; a connection whose weights are not here yet, or whose kind + this build cannot run, is 409 and names what to do; a connection whose model + answers words rather than places is 422. + """ + prompt = PointPrompt( + positive=tuple((point.x, point.y) for point in body.positive), + negative=tuple((point.x, point.y) for point in body.negative), + ) + prediction = suggest( + workspace, + project_id=body.project_id, + asset_id=body.asset_id, + connection_id=body.connection_id, + prompt=prompt, + allowed=tuple(body.allowed_geometries), + detail=DEFAULT_DETAIL if body.detail is None else body.detail, + ) + region = next(iter(prediction.regions), None) + return SuggestionOut( + model_ref=prediction.model_ref, + region=None + if region is None + else SuggestedRegion(geometry=region.geometry, confidence=region.confidence), + ) + + @router.delete( "/{connection_id}", status_code=status.HTTP_204_NO_CONTENT, diff --git a/tests/inference/stubs.py b/tests/inference/stubs.py new file mode 100644 index 00000000..b64d7a3d --- /dev/null +++ b/tests/inference/stubs.py @@ -0,0 +1,182 @@ +"""Stand-ins for the parts of the optional runtime the SAM adapter reaches for. + +Stubs rather than mocks, on ``test_fp16``'s terms: every attribute here is one +the adapter genuinely touches, so the shape of this file is a readable statement +of what the adapter depends on. If it grows, the coupling grew — and the point of +the adapter being written the way it is, is that this file stays small enough to +read. + +Nothing here imports torch, which is what lets the whole point-prompt path be +exercised on a machine with no GPU and no ``local-inference`` extra installed. +""" + +from __future__ import annotations + +from types import SimpleNamespace +from typing import Any + +Mask = list[list[bool]] + + +class Values: + """A tensor-shaped thing holding one list of numbers.""" + + def __init__(self, values: list[float]) -> None: + self._values = values + + def flatten(self) -> Values: + return self + + def tolist(self) -> list[float]: + return self._values + + +class Grid: + """One mask, and the only thing the adapter asks of it.""" + + def __init__(self, mask: Mask) -> None: + self._mask = mask + + def tolist(self) -> Mask: + return self._mask + + +class Stack: + """Several masks, addressed the way ``post_process_masks`` output is. + + The adapter reshapes away the batch and object dimensions and then indexes, + which is exactly the two operations reproduced here — deliberately no more, + so that a change in how the adapter reads its output shows up as an + ``AttributeError`` here rather than as a silently different answer. + """ + + def __init__(self, masks: list[Mask]) -> None: + self._masks = masks + + @property + def shape(self) -> tuple[int, int, int, int]: + return (1, len(self._masks), len(self._masks[0]), len(self._masks[0][0])) + + def reshape(self, *_: Any) -> Stack: + return self + + def __getitem__(self, index: int) -> Grid: + return Grid(self._masks[index]) + + +class Inputs(dict[str, Any]): + """What a processor hands back: a mapping that also knows how to move device.""" + + def to(self, device: str) -> Inputs: + self.device = device + return self + + +class StubProcessor: + """Both halves of the encode/decode split, and a record of which was asked for. + + ``encodes`` counts calls carrying an image and ``decodes`` counts calls + carrying points — the two the adapter deliberately keeps apart, and the only + externally visible difference between a working embedding cache and a + bypassed one. + """ + + def __init__(self, masks: list[Mask], scores: list[float]) -> None: + self._masks = masks + self._scores = scores + self.encodes = 0 + self.decodes = 0 + self.post_processed: list[list[list[int]]] = [] + + def __call__(self, **kwargs: Any) -> Inputs: + if kwargs.get("images") is not None: + self.encodes += 1 + return Inputs(pixel_values="pixels") + self.decodes += 1 + return Inputs( + input_points=kwargs["input_points"], + input_labels=kwargs["input_labels"], + original_sizes=kwargs["original_sizes"], + ) + + def post_process_masks( + self, masks: Any, original_sizes: list[list[int]], binarize: bool = True + ) -> list[Stack]: + self.post_processed.append(original_sizes) + return [Stack(self._masks)] + + +class StubModel: + """A segmenter that answers from a fixed script, and counts its encodes.""" + + def __init__(self, masks: list[Mask], scores: list[float]) -> None: + self._masks = masks + self._scores = scores + self.encodes = 0 + self.prompts: list[tuple[Any, Any]] = [] + self.embeddings_seen: list[Any] = [] + + def get_image_embeddings(self, pixel_values: Any) -> str: + self.encodes += 1 + return f"embedding-{self.encodes}" + + def __call__( + self, + *, + input_points: Any, + input_labels: Any, + image_embeddings: Any, + multimask_output: bool, + ) -> SimpleNamespace: + self.prompts.append((input_points, input_labels)) + self.embeddings_seen.append(image_embeddings) + return SimpleNamespace(pred_masks=Stack(self._masks), iou_scores=Values(self._scores)) + + +class Functional: + """The one torch function the fp16 guard swaps out and puts back.""" + + @staticmethod + def grid_sample(input: Any, grid: Any, *args: Any, **kwargs: Any) -> str: + return "sampled" + + +class _Scope: + def __enter__(self) -> None: + return None + + def __exit__(self, *_: Any) -> None: + return None + + +class StubTorch: + """Enough of torch for ``forward_guard`` to do its work and restore itself.""" + + float16 = "float16" + + def __init__(self) -> None: + self.nn = SimpleNamespace(functional=Functional()) + + def no_grad(self) -> _Scope: + return _Scope() + + def autocast(self, device_type: str, dtype: str) -> _Scope: + return _Scope() + + class cuda: # noqa: N801 — mirrors torch's own spelling + @staticmethod + def is_available() -> bool: + return False + + +def disc(radius: int, *, size: int = 64) -> Mask: + """A filled circle — a mask with an outline worth simplifying.""" + centre = size // 2 + return [ + [(x - centre) ** 2 + (y - centre) ** 2 <= radius * radius for x in range(size)] + for y in range(size) + ] + + +def blank(size: int = 64) -> Mask: + return [[False] * size for _ in range(size)] diff --git a/tests/inference/test_cache.py b/tests/inference/test_cache.py new file mode 100644 index 00000000..0ecd2094 --- /dev/null +++ b/tests/inference/test_cache.py @@ -0,0 +1,90 @@ +"""The bounded LRU both caches are built on. + +Small enough to test by hand, and worth testing by hand: it is the thing standing +between the suggest route and a model load per click. +""" + +from __future__ import annotations + +import pytest + +from visionset.inference.cache import ( + DEFAULT_EMBEDDING_CAPACITY, + DEFAULT_PROVIDER_CAPACITY, + BoundedCache, +) + + +def test_what_goes_in_comes_out() -> None: + cache: BoundedCache[str, int] = BoundedCache(2) + cache.put("a", 1) + assert cache.get("a") == 1 + assert cache.get("missing") is None + + +def test_the_bound_is_respected_and_the_oldest_goes_first() -> None: + cache: BoundedCache[str, int] = BoundedCache(2) + cache.put("a", 1) + cache.put("b", 2) + cache.put("c", 3) + assert len(cache) == 2 + assert "a" not in cache + assert "b" in cache and "c" in cache + + +def test_reading_something_keeps_it_alive() -> None: + """LRU rather than first-in-first-out, which is the whole reason for the class. + + The asset somebody is clicking on repeatedly is the one that must survive, + and under insertion order it is the one that would be evicted first. + """ + cache: BoundedCache[str, int] = BoundedCache(2) + cache.put("a", 1) + cache.put("b", 2) + cache.get("a") + cache.put("c", 3) + assert "a" in cache, "used most recently" + assert "b" not in cache + + +def test_membership_does_not_count_as_a_use() -> None: + """Otherwise a test asserting eviction would change the thing it is measuring.""" + cache: BoundedCache[str, int] = BoundedCache(2) + cache.put("a", 1) + cache.put("b", 2) + assert "a" in cache + cache.put("c", 3) + assert "a" not in cache + + +def test_writing_a_key_again_refreshes_it_rather_than_growing() -> None: + cache: BoundedCache[str, int] = BoundedCache(2) + cache.put("a", 1) + cache.put("b", 2) + cache.put("a", 9) + cache.put("c", 3) + assert len(cache) == 2 + assert cache.get("a") == 9 + assert "b" not in cache + + +def test_discarding_and_clearing() -> None: + cache: BoundedCache[str, int] = BoundedCache(4) + cache.put("a", 1) + cache.discard("a") + cache.discard("a") # a no-op the second time rather than a KeyError + assert len(cache) == 0 + cache.put("b", 2) + cache.clear() + assert len(cache) == 0 + + +def test_a_cache_that_holds_nothing_is_refused() -> None: + with pytest.raises(ValueError, match="not a cache"): + BoundedCache(0) + + +def test_the_shipped_capacities_leave_room_for_the_co_residency_the_design_assumes() -> None: + """Two providers is a detector and a segmenter, which is what D1 describes.""" + assert DEFAULT_PROVIDER_CAPACITY >= 2 + assert DEFAULT_EMBEDDING_CAPACITY >= 2 diff --git a/tests/inference/test_masks.py b/tests/inference/test_masks.py new file mode 100644 index 00000000..ec98a95d --- /dev/null +++ b/tests/inference/test_masks.py @@ -0,0 +1,169 @@ +"""Mask to geometry: the conversion D3 lives in, driven with literals. + +No torch anywhere here, which is the point of ``masks`` being written over plain +sequences: the part of a segmentation adapter that can be wrong about *shape* is +provable on any machine, and the part that needs a GPU is the part that produces +the booleans rather than the part that reads them. +""" + +from __future__ import annotations + +import pytest + +from visionset.inference.masks import ( + DEFAULT_DETAIL, + bbox_from, + bounds_of, + narrowed, + outline, + polygon_from, + simplified, + spans, +) +from visionset.kernel.domain import BboxGeometry, GeometryType, PolygonGeometry + + +def disc(radius: int, *, width: int | None = None, height: int | None = None) -> list[list[bool]]: + """A filled circle: the stand-in for an organic shape, and what D3's band was written for.""" + width = width or 2 * radius + 8 + height = height or 2 * radius + 8 + cx, cy = width // 2, height // 2 + return [ + [(x - cx) ** 2 + (y - cy) ** 2 <= radius * radius for x in range(width)] + for y in range(height) + ] + + +def rect( + x0: int, y0: int, x1: int, y1: int, *, width: int = 100, height: int = 100 +) -> list[list[bool]]: + return [[x0 <= x <= x1 and y0 <= y <= y1 for x in range(width)] for y in range(height)] + + +def empty(size: int = 10) -> list[list[bool]]: + return [[False] * size for _ in range(size)] + + +# --- the extent --------------------------------------------------------------- + + +def test_a_box_is_the_pixels_outer_edge() -> None: + """Inclusive of the last lit pixel, so a 30-wide run is 30 wide and not 29.""" + assert bbox_from(rect(10, 20, 39, 49)) == BboxGeometry(x=10.0, y=20.0, width=30.0, height=30.0) + + +def test_one_lit_pixel_is_a_box_one_unit_across() -> None: + """Not zero-area: the domain refuses that, and one pixel is a real thing to point at.""" + assert bbox_from(rect(5, 5, 5, 5)) == BboxGeometry(x=5.0, y=5.0, width=1.0, height=1.0) + + +def test_an_empty_mask_has_no_box() -> None: + """A click on sky is an ordinary thing to do, and None is what it answers.""" + assert bbox_from(empty()) is None + assert spans(empty()) == [] + + +# --- the outline -------------------------------------------------------------- + + +def test_the_outline_closes_on_itself() -> None: + traced = outline(rect(10, 10, 20, 20)) + assert traced[0] == (10.0, 10.0) + assert len(traced) == 40 # the perimeter of an 11x11 square, corners counted once + assert len(set(traced)) == len(traced), "no pixel is walked twice" + + +def test_an_isolated_pixel_has_no_ring_to_walk() -> None: + assert outline(rect(3, 3, 3, 3)) == [(3.0, 3.0)] + + +# --- simplification ----------------------------------------------------------- + + +def test_simplification_keeps_the_corners_and_drops_the_straight_runs() -> None: + line = [(0.0, 0.0), (1.0, 0.0), (2.0, 0.0), (3.0, 0.0), (3.0, 1.0)] + assert simplified(line, tolerance=0.5) == [(0.0, 0.0), (3.0, 0.0), (3.0, 1.0)] + + +def test_a_bigger_tolerance_keeps_less() -> None: + bumpy = [(float(x), 1.0 if x % 2 else 0.0) for x in range(20)] + assert len(simplified(bumpy, tolerance=0.1)) > len(simplified(bumpy, tolerance=2.0)) + + +# --- D3's vertex band --------------------------------------------------------- + + +@pytest.mark.parametrize("radius", [8, 15, 30, 60, 120, 300]) +def test_a_typical_object_lands_in_the_ten_to_forty_vertex_band(radius: int) -> None: + """D3's range, and the property that says the tolerance is relative rather than absolute. + + The same detail setting has to work on a thing eight pixels across and a + thing six hundred across, which an absolute pixel tolerance cannot do: three + pixels is nothing on a car and is the whole of a bottle cap. Asserting the + band across a 37x size range is what would fail if the tolerance stopped + scaling with the region. + """ + polygon = polygon_from(disc(radius)) + assert polygon is not None + assert 10 <= len(polygon.points) <= 40 + + +def test_a_rectangle_comes_back_as_exactly_its_corners() -> None: + """The closing artifact, pinned. + + Douglas-Peucker pins the last point of what it is given, and what it is given + is a ring cut open at an arbitrary pixel — so the final vertex is pinned for + a reason that stops being true once the ring closes, landing one pixel from + the first. This asserts the near-duplicate is gone, which an equality check + on first-versus-last would never catch, because it is not a duplicate. + """ + polygon = polygon_from(rect(10, 10, 60, 60)) + assert polygon is not None + assert polygon.points == [(10.0, 10.0), (60.0, 10.0), (60.0, 60.0), (10.0, 60.0)] + + +def test_a_shape_too_thin_to_be_a_polygon_is_refused() -> None: + """Two points are a line. The domain wants three, and a caller wants a shape worth accepting.""" + assert polygon_from(rect(10, 10, 11, 10)) is None + assert polygon_from(empty()) is None + + +def test_more_detail_means_more_vertices() -> None: + fine = polygon_from(disc(60), detail=DEFAULT_DETAIL / 4) + coarse = polygon_from(disc(60), detail=DEFAULT_DETAIL * 4) + assert fine is not None and coarse is not None + assert len(fine.points) > len(coarse.points) + + +# --- narrowing to what the class admits (D3) ----------------------------------- + + +def test_a_polygon_stands_where_polygons_are_allowed() -> None: + polygon = PolygonGeometry(points=[(0.0, 0.0), (10.0, 0.0), (10.0, 10.0)]) + assert narrowed(polygon, allowed=[GeometryType.POLYGON, GeometryType.BBOX]) is polygon + + +def test_a_polygon_becomes_its_own_box_where_only_boxes_are_allowed() -> None: + """The D3 fallback, and the assertion the mutation test in the PR body breaks.""" + polygon = PolygonGeometry(points=[(2.0, 3.0), (12.0, 3.0), (12.0, 9.0), (2.0, 9.0)]) + assert narrowed(polygon, allowed=[GeometryType.BBOX]) == BboxGeometry( + x=2.0, y=3.0, width=10.0, height=6.0 + ) + + +def test_a_class_admitting_neither_is_offered_nothing() -> None: + """D3's third case: the gesture is not offered for a tag-only class at all.""" + polygon = PolygonGeometry(points=[(0.0, 0.0), (10.0, 0.0), (10.0, 10.0)]) + assert narrowed(polygon, allowed=[GeometryType.CLASSIFICATION_TAG]) is None + + +def test_a_box_is_never_widened_into_a_polygon() -> None: + """Narrowing only ever loses information. A box cannot become the outline it never held.""" + box = BboxGeometry(x=0.0, y=0.0, width=4.0, height=4.0) + assert narrowed(box, allowed=[GeometryType.POLYGON]) is None + assert narrowed(box, allowed=[GeometryType.BBOX]) is box + + +def test_a_flat_outline_still_yields_a_storable_box() -> None: + """Zero height is widened rather than refused — the domain will not store a degenerate box.""" + assert bounds_of([(0.0, 5.0), (9.0, 5.0)]) == BboxGeometry(x=0.0, y=5.0, width=9.0, height=1.0) diff --git a/tests/inference/test_providers.py b/tests/inference/test_providers.py new file mode 100644 index 00000000..b5680b6e --- /dev/null +++ b/tests/inference/test_providers.py @@ -0,0 +1,207 @@ +"""Resolution and residency: which adapter answers, and how often one is built. + +The refusals here are the ones a caller can act on, and each names what to do. +The pool is what makes the embedding cache inside a provider worth anything — +a provider rebuilt per request carries an empty cache into every click. +""" + +from __future__ import annotations + +from collections.abc import Iterator +from pathlib import Path +from typing import Any + +import pytest + +from visionset.inference import providers as providers_module +from visionset.inference.providers import ( + SEGMENTER_FAMILIES, + ProviderPool, + family_of, + provider_for, + resident, +) +from visionset.inference.sam_provider import LocalSamProvider +from visionset.inference.transformers_provider import LocalTransformersProvider +from visionset.kernel.domain import ConnectionType, InferenceConnection +from visionset.kernel.errors import ( + InferenceConnectionNotRunnable, + InferenceConnectionNotSetUp, + LocalInferenceUnavailable, +) +from visionset.kernel.services import InferenceConnectionService, WorkspaceService + + +@pytest.fixture() +def workspace(tmp_path: Path) -> Iterator[WorkspaceService]: + made = WorkspaceService.init(tmp_path / "ws", name="providers") + try: + yield made + finally: + made.close() + + +@pytest.fixture() +def connections(workspace: WorkspaceService) -> InferenceConnectionService: + return InferenceConnectionService(workspace) + + +def a_local( + connections: InferenceConnectionService, name: str = "seg", *, ready: bool = True +) -> InferenceConnection: + made = connections.create( + name, + connection_type=ConnectionType.LOCAL, + model_id="some/segmenter", + model_revision="abc123", + device="cpu", + precision="fp32", + ) + return connections.record_weights_ready(made.id) if ready else made + + +def an_http(connections: InferenceConnectionService) -> InferenceConnection: + return connections.create( + "hosted", + connection_type=ConnectionType.HTTP, + model_id="some/model", + model_revision="v1", + endpoint_url="https://example.invalid/predict", + ) + + +def no_extra_needed(monkeypatch: pytest.MonkeyPatch, family: str) -> None: + """Pretend the optional runtime is installed and declares that family.""" + monkeypatch.setattr(providers_module, "require", lambda: None) + monkeypatch.setattr(providers_module, "family_of", lambda *_, **__: family) + + +# --- refusals ----------------------------------------------------------------- + + +def test_a_connection_without_weights_is_told_which_action_fixes_it( + connections: InferenceConnectionService, tmp_path: Path +) -> None: + connection = a_local(connections, ready=False) + with pytest.raises(InferenceConnectionNotSetUp, match="download_weights"): + provider_for(connection, workspace_root=tmp_path) + + +def test_an_http_connection_is_refused_because_this_build_has_no_adapter( + connections: InferenceConnectionService, tmp_path: Path +) -> None: + with pytest.raises(InferenceConnectionNotRunnable, match="http connection"): + provider_for(an_http(connections), workspace_root=tmp_path) + + +def test_a_missing_runtime_is_reported_after_the_connections_own_state( + connections: InferenceConnectionService, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Order matters: "your weights are not here" is fixable from where the caller stands.""" + + def absent() -> None: + raise LocalInferenceUnavailable('install it with: pip install "visionset[local-inference]"') + + monkeypatch.setattr(providers_module, "require", absent) + with pytest.raises(LocalInferenceUnavailable, match="local-inference"): + provider_for(a_local(connections), workspace_root=tmp_path) + + # ...and the not-set-up connection still gets its own answer rather than this one. + with pytest.raises(InferenceConnectionNotSetUp): + provider_for(a_local(connections, "other", ready=False), workspace_root=tmp_path) + + +# --- which family answers ----------------------------------------------------- + + +def test_a_segmenter_config_resolves_to_the_point_prompt_adapter( + connections: InferenceConnectionService, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + no_extra_needed(monkeypatch, "sam2") + assert isinstance(provider_for(a_local(connections), workspace_root=tmp_path), LocalSamProvider) + + +def test_anything_else_resolves_to_the_detector( + connections: InferenceConnectionService, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """The older and more common case, and the fallback for a config nothing could read.""" + no_extra_needed(monkeypatch, "grounding-dino") + assert isinstance( + provider_for(a_local(connections), workspace_root=tmp_path), LocalTransformersProvider + ) + + +def test_an_unreadable_config_answers_empty_rather_than_raising( + connections: InferenceConnectionService, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A load that is going to fail should fail with the library's message, not this one's.""" + + class Broken: + class AutoConfig: + @staticmethod + def from_pretrained(*_: Any, **__: Any) -> Any: + raise OSError("nothing in the cache") + + monkeypatch.setattr(providers_module, "imported", lambda _: Broken()) + assert family_of(a_local(connections), cache_dir=tmp_path) == "" + + +def test_the_family_names_are_a_set_so_the_video_variant_is_one_more_member() -> None: + """D1 keeps the 0.2.0 door open; it opens as a name here, not a second mechanism.""" + assert "sam2" in SEGMENTER_FAMILIES + + +# --- residency ---------------------------------------------------------------- + + +def test_asking_twice_for_the_same_connection_builds_one_provider( + connections: InferenceConnectionService, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + no_extra_needed(monkeypatch, "sam2") + connection = a_local(connections) + pool = ProviderPool() + + first = pool.get(connection, workspace_root=tmp_path) + second = pool.get(connection, workspace_root=tmp_path) + + assert first is second + assert pool.builds == 1 + + +def test_editing_a_connection_builds_a_new_provider( + connections: InferenceConnectionService, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Changing the model id, the device or the precision must not keep serving the old one.""" + no_extra_needed(monkeypatch, "sam2") + connection = a_local(connections) + pool = ProviderPool() + pool.get(connection, workspace_root=tmp_path) + + edited = connections.update(connection.id, device="cuda") + pool.get(edited, workspace_root=tmp_path) + + assert pool.builds == 2 + + +def test_the_pool_is_bounded( + connections: InferenceConnectionService, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + no_extra_needed(monkeypatch, "sam2") + pool = ProviderPool(capacity=1) + pool.get(a_local(connections, "one"), workspace_root=tmp_path) + pool.get(a_local(connections, "two"), workspace_root=tmp_path) + assert len(pool) == 1 + + +def test_a_refused_connection_leaves_nothing_behind_for_the_next_request( + connections: InferenceConnectionService, tmp_path: Path +) -> None: + pool = ProviderPool() + with pytest.raises(InferenceConnectionNotSetUp): + pool.get(a_local(connections, ready=False), workspace_root=tmp_path) + assert len(pool) == 0 + assert pool.builds == 0 + + +def test_the_process_wide_pool_is_one_object() -> None: + assert resident() is resident() diff --git a/tests/inference/test_sam_provider.py b/tests/inference/test_sam_provider.py new file mode 100644 index 00000000..745021f6 --- /dev/null +++ b/tests/inference/test_sam_provider.py @@ -0,0 +1,213 @@ +"""The point-prompt adapter: what it refuses, what it answers, and what it caches. + +Every test here runs with no torch, no GPU and no ``local-inference`` extra — +the adapter's own seams (``_ready``, and ``imported`` for torch) are the two +places a stand-in goes in, and the rest of the path is the shipped code. +""" + +from __future__ import annotations + +from pathlib import Path +from typing import Any +from uuid import uuid4 + +import pytest +from tests.inference.stubs import StubModel, StubProcessor, StubTorch, blank, disc + +from visionset.inference import sam_provider +from visionset.inference.sam_provider import ( + NEGATIVE, + POSITIVE, + LocalSamProvider, + best_of, + points_and_labels, +) +from visionset.kernel.domain import ( + PointPrompt, + PolygonGeometry, + PredictionRequest, + PredictionTarget, + TextPrompt, +) +from visionset.kernel.errors import UnsupportedPrompt + +PNG = ( + b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x01\x00\x00\x00\x01\x08\x06" + b"\x00\x00\x00\x1f\x15\xc4\x89\x00\x00\x00\nIDATx\x9cc\x00\x01\x00\x00\x05\x00" + b"\x01\r\n-\xb4\x00\x00\x00\x00IEND\xaeB`\x82" +) +"""A real one-pixel PNG, because the adapter genuinely decodes what it is handed.""" + + +def built( + monkeypatch: pytest.MonkeyPatch, + *, + masks: list[list[list[bool]]] | None = None, + scores: list[float] | None = None, +) -> tuple[LocalSamProvider, StubProcessor, StubModel]: + """A provider whose model is a script, with everything else as shipped.""" + processor = StubProcessor(masks or [disc(20)], scores or [0.9]) + model = StubModel(masks or [disc(20)], scores or [0.9]) + provider = LocalSamProvider( + "some/segmenter", + "abc123", + device="cpu", + precision=None, + cache_dir=Path("/nowhere"), + connection_name="local", + ) + monkeypatch.setattr(provider, "_ready", lambda: (processor, model, "cpu", False)) + monkeypatch.setattr(sam_provider, "imported", lambda _: StubTorch()) + return provider, processor, model + + +def target(asset_id: Any = None) -> PredictionTarget: + return PredictionTarget(asset_id=asset_id or uuid4(), content=PNG, media_type="image/png") + + +def asked(prompt: PointPrompt, *targets: PredictionTarget) -> PredictionRequest: + return PredictionRequest(targets=targets or (target(),), prompt=prompt) + + +def one_click() -> PointPrompt: + return PointPrompt(positive=((10.0, 12.0),)) + + +# --- the prompt conversion ---------------------------------------------------- + + +def test_positives_come_first_and_each_point_gets_its_own_label() -> None: + points, labels = points_and_labels( + PointPrompt(positive=((1.0, 2.0), (3.0, 4.0)), negative=((5.0, 6.0),)) + ) + assert points == [[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]] + assert labels == [POSITIVE, POSITIVE, NEGATIVE] + + +def test_a_prompt_with_no_negatives_still_labels_every_point() -> None: + points, labels = points_and_labels(one_click()) + assert len(points) == len(labels) == 1 + + +# --- choosing among the multi-mask answers ------------------------------------ + + +def test_the_highest_scoring_mask_is_the_one_offered() -> None: + """A click is ambiguous about scale; the model's own IoU head is what resolves it.""" + assert best_of([0.2, 0.91, 0.5], [[], [], []]) == (1, 0.91) + + +def test_a_score_outside_the_domains_bounds_is_clamped_rather_than_refused() -> None: + assert best_of([1.0000001], [[]])[1] == 1.0 + + +# --- what it refuses ---------------------------------------------------------- + + +def test_a_text_prompt_is_refused_because_this_model_answers_places( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The counterpart of the detector adapter's refusal, which stays as it is.""" + provider, _, _ = built(monkeypatch) + with pytest.raises(UnsupportedPrompt, match="point prompts"): + list(provider.predict(asked(TextPrompt(phrases=("cat",))))) # type: ignore[arg-type] + + +# --- what it answers ---------------------------------------------------------- + + +def test_a_click_comes_back_as_one_polygon_carrying_the_models_confidence( + monkeypatch: pytest.MonkeyPatch, +) -> None: + provider, _, _ = built(monkeypatch, masks=[disc(20)], scores=[0.87]) + (answer,) = list(provider.predict(asked(one_click()))) + + assert answer.model_ref == "some/segmenter@abc123" + (region,) = answer.regions + assert isinstance(region.geometry, PolygonGeometry) + assert region.confidence == pytest.approx(0.87) + + +def test_the_label_is_empty_because_pointing_says_where_and_not_what( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """The editor already knows the active class; a name invented here would be a worse copy.""" + provider, _, _ = built(monkeypatch) + (answer,) = list(provider.predict(asked(one_click()))) + assert answer.regions[0].label == "" + + +def test_an_empty_mask_is_an_ordinary_answer_with_nothing_in_it( + monkeypatch: pytest.MonkeyPatch, +) -> None: + provider, _, _ = built(monkeypatch, masks=[blank()], scores=[0.9]) + (answer,) = list(provider.predict(asked(one_click()))) + assert answer.regions == () + + +def test_a_model_less_sure_than_the_caller_asked_answers_nothing( + monkeypatch: pytest.MonkeyPatch, +) -> None: + provider, _, _ = built(monkeypatch, masks=[disc(20)], scores=[0.3]) + request = PredictionRequest(targets=(target(),), prompt=one_click(), minimum_confidence=0.8) + (answer,) = list(provider.predict(request)) + assert answer.regions == () + assert answer.model_ref == "some/segmenter@abc123", "still says who was asked" + + +def test_negative_points_reach_the_model_alongside_the_positive_ones( + monkeypatch: pytest.MonkeyPatch, +) -> None: + provider, _, model = built(monkeypatch) + prompt = PointPrompt(positive=((10.0, 12.0),), negative=((30.0, 30.0),)) + list(provider.predict(asked(prompt))) + (points, labels) = model.prompts[0] + assert points == [[[[10.0, 12.0], [30.0, 30.0]]]] + assert labels == [[[POSITIVE, NEGATIVE]]] + + +# --- the embedding cache (D5) ------------------------------------------------- + + +def test_a_second_click_on_the_same_asset_decodes_without_encoding_again( + monkeypatch: pytest.MonkeyPatch, +) -> None: + """D5's encode-once behaviour, and the test the PR body names for the cache mutation. + + Turn the cache off — return the computed embedding without holding it — and + ``encodes`` becomes 2 while every assertion about the *answer* stays true. + That is exactly why the counter exists: correctness cannot tell these two + implementations apart, and the latency budget is the whole feature. + """ + provider, processor, model = built(monkeypatch) + asset = uuid4() + + list(provider.predict(asked(one_click(), target(asset)))) + list(provider.predict(asked(PointPrompt(positive=((11.0, 13.0),)), target(asset)))) + + assert provider.encodes == 1, "the image is read once" + assert model.encodes == 1 + assert processor.encodes == 1 + assert processor.decodes == 2, "but both clicks were answered" + assert model.embeddings_seen == ["embedding-1", "embedding-1"] + + +def test_a_different_asset_pays_its_own_encode(monkeypatch: pytest.MonkeyPatch) -> None: + provider, _, model = built(monkeypatch) + list(provider.predict(asked(one_click(), target()))) + list(provider.predict(asked(one_click(), target()))) + assert provider.encodes == 2 + assert model.embeddings_seen == ["embedding-1", "embedding-2"] + + +def test_the_cache_is_bounded_and_evicts_the_least_recently_used( + monkeypatch: pytest.MonkeyPatch, +) -> None: + provider, _, _ = built(monkeypatch) + monkeypatch.setattr(provider, "_embeddings", type(provider._embeddings)(2)) + first, second, third = target(), target(), target() + + for one in (first, second, third, first): + list(provider.predict(asked(one_click(), one))) + + assert provider.encodes == 4, "the first asset was evicted by the third and re-encoded" diff --git a/tests/server/test_downloads.py b/tests/server/test_downloads.py index 00f8a404..d89d5329 100644 --- a/tests/server/test_downloads.py +++ b/tests/server/test_downloads.py @@ -23,9 +23,8 @@ from tests.server._flow import batch_from_ingest, project_with_schema from tests.server._jobs import InlineDispatcher -from visionset.kernel.domain import ImageFormat +from visionset.kernel.domain import MEDIA_TYPES, ImageFormat from visionset.kernel.ports import THUMBNAIL_FORMAT -from visionset.server.routes.assets import _MEDIA_TYPES @pytest.fixture() @@ -61,18 +60,23 @@ def ingested( def test_every_image_format_has_a_media_type() -> None: - """Indexed directly by the route, so a new member must arrive with its type. + """Indexed directly by every reader, so a new member must arrive with its type. Read off the enum rather than restated, the `ProgressCounts` bargain: adding a format without a media type fails here instead of quietly degrading every download of it. + + The table lives in the domain rather than in this route since the inference + adapters became its second reader — a provider has to be told what the bytes + it is handed are — so this asserts a domain fact from the surface that first + needed it. """ - assert set(_MEDIA_TYPES) == set(ImageFormat) + assert set(MEDIA_TYPES) == set(ImageFormat) def test_the_thumbnail_format_is_one_of_them() -> None: """The route indexes the table with it, so it cannot be a format nobody mapped.""" - assert THUMBNAIL_FORMAT in _MEDIA_TYPES + assert THUMBNAIL_FORMAT in MEDIA_TYPES # --- the asset itself --------------------------------------------------------- diff --git a/tests/server/test_suggest.py b/tests/server/test_suggest.py new file mode 100644 index 00000000..10ede057 --- /dev/null +++ b/tests/server/test_suggest.py @@ -0,0 +1,320 @@ +"""The suggest route: what it refuses, in what order, and what a suggestion looks like. + +The model is a stand-in — running a real segmenter is neither a unit test nor a +thing CI should do — but everything else on the path is shipped code: the route, +the orchestration, the narrowing to what the class admits, and the error +translation. + +The refusal tests deliberately send no asset at all. That is not a shortcut: the +orchestration resolves the connection *before* it looks the asset up, precisely +so somebody part-way through setting a connection up is told about the +connection rather than about an asset that was never the problem. These tests are +what holds that order in place. +""" + +from __future__ import annotations + +from collections.abc import Iterator +from pathlib import Path +from typing import Any +from uuid import uuid4 + +import pytest +from fastapi.testclient import TestClient +from tests.fixtures.media import write_image +from tests.server._api import api_client +from tests.server._jobs import InlineDispatcher + +from visionset.inference import suggestions as suggestions_module +from visionset.inference import weights as weights_module +from visionset.kernel.domain import AssetPrediction, PolygonGeometry, PredictedRegion +from visionset.server.routes import inference as inference_routes + + +@pytest.fixture(autouse=True) +def downloadable(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None: + """Let a connection genuinely reach ``ready`` without a network or the extra. + + The two seams ``test_inference`` already uses: the route's install check, and + the one function that would touch the network. Everything between them — + the gate, the ordering, the write that records the connection ready — is the + shipped code, so a connection that says it is ready here got there the way a + real one does. Without this the helper below would quietly leave every + connection ``not_set_up`` and the happy-path tests would be passing on a + stub that hid a 409. + """ + monkeypatch.setattr(inference_routes, "require_local_inference", lambda: None) + monkeypatch.setattr(weights_module, "download", lambda connection, *, into: into) + + +@pytest.fixture() +def runner() -> InlineDispatcher: + return InlineDispatcher() + + +@pytest.fixture() +def client(tmp_path: Path, runner: InlineDispatcher) -> Iterator[TestClient]: + with api_client(tmp_path / "ws", dispatcher=runner) as made: + yield made + + +@pytest.fixture() +def project(client: TestClient) -> str: + made = client.post("/projects", json={"name": "suggesting"}) + return str(made.json()["id"]) + + +def a_connection(client: TestClient, *, kind: str = "local", ready: bool = True) -> str: + body: dict[str, Any] = ( + { + "name": f"c-{uuid4().hex[:6]}", + "connection_type": "local", + "model_id": "some/segmenter", + "model_revision": "abc123", + "device": "cpu", + "precision": "fp32", + } + if kind == "local" + else { + "name": f"c-{uuid4().hex[:6]}", + "connection_type": "http", + "model_id": "some/model", + "model_revision": "v1", + "endpoint_url": "https://example.invalid/predict", + } + ) + made = client.post("/inference/connections", json=body).json() + if kind == "local" and ready: + queued = client.post(f"/inference/connections/{made['id']}/download") + assert queued.status_code == 202, queued.text + assert ( + client.get(f"/inference/connections/{made['id']}").json()["setup_state"] == "ready" + ), "the helper must leave a connection genuinely ready, not merely asked" + return str(made["id"]) + + +def an_asset(client: TestClient, runner: InlineDispatcher, project: str, tmp_path: Path) -> str: + write_image(tmp_path / "one.png") + with (tmp_path / "one.png").open("rb") as handle: + source = client.post( + f"/projects/{project}/sources/images", + files=[("files", ("one.png", handle, "image/png"))], + ).json() + client.post(f"/sources/{source['id']}/ingest-jobs", json={}) + runner.wait() + return str(client.get(f"/projects/{project}/assets").json()["items"][0]["id"]) + + +def ask( + client: TestClient, + *, + project: str, + asset: str, + connection: str, + allowed: list[str] | None = None, + positive: list[dict[str, float]] | None = None, + negative: list[dict[str, float]] | None = None, +) -> Any: + return client.post( + "/inference/suggest", + json={ + "project_id": project, + "asset_id": asset, + "connection_id": connection, + "positive": [{"x": 32.0, "y": 32.0}] if positive is None else positive, + "negative": negative or [], + "allowed_geometries": ["polygon"] if allowed is None else allowed, + }, + ) + + +@pytest.fixture() +def answering(monkeypatch: pytest.MonkeyPatch) -> list[Any]: + """A provider that answers from a script, installed where the route resolves one.""" + asked: list[Any] = [] + + class Provider: + model_ref = "some/segmenter@abc123" + + def predict(self, request: Any) -> Any: + asked.append(request) + polygon = PolygonGeometry(points=[(2.0, 3.0), (12.0, 3.0), (12.0, 9.0), (2.0, 9.0)]) + yield AssetPrediction( + asset_id=request.targets[0].asset_id, + model_ref=self.model_ref, + regions=(PredictedRegion(label="", confidence=0.82, geometry=polygon),), + ) + + class Pool: + def get(self, connection: Any, *, workspace_root: Path) -> Any: + return Provider() + + monkeypatch.setattr(suggestions_module, "resident", Pool) + return asked + + +# --- refusals, and their order ------------------------------------------------ + + +def test_an_unknown_connection_is_not_found(client: TestClient, project: str) -> None: + answer = ask(client, project=project, asset=str(uuid4()), connection=str(uuid4())) + assert answer.status_code == 404 + assert answer.json()["code"] == "INFERENCE_CONNECTION_NOT_FOUND" + + +def test_a_connection_without_weights_is_refused_with_what_to_do( + client: TestClient, project: str +) -> None: + """409 rather than 500: the caller changes something and resubmits.""" + connection = a_connection(client, ready=False) + answer = ask(client, project=project, asset=str(uuid4()), connection=connection) + assert answer.status_code == 409 + assert answer.json()["code"] == "INFERENCE_CONNECTION_NOT_SET_UP" + assert "download_weights" in answer.json()["message"] + + +def test_an_http_connection_says_this_build_cannot_run_it(client: TestClient, project: str) -> None: + connection = a_connection(client, kind="http") + answer = ask(client, project=project, asset=str(uuid4()), connection=connection) + assert answer.json()["code"] == "INFERENCE_CONNECTION_NOT_RUNNABLE" + + +def test_the_connection_is_resolved_before_the_asset(client: TestClient, project: str) -> None: + """A connection problem and a nonexistent asset together answer about the connection. + + The order the orchestration documents, pinned: somebody part-way through + setting a connection up should not be told their asset is missing instead. + """ + connection = a_connection(client, ready=False) + answer = ask(client, project=project, asset=str(uuid4()), connection=connection) + assert answer.json()["code"] == "INFERENCE_CONNECTION_NOT_SET_UP" + + +def test_an_unknown_asset_is_not_found_once_the_connection_is_fine( + client: TestClient, project: str, answering: list[Any] +) -> None: + connection = a_connection(client) + answer = ask(client, project=project, asset=str(uuid4()), connection=connection) + assert answer.status_code == 404 + assert answer.json()["code"] == "ASSET_NOT_FOUND" + + +def test_a_gesture_with_no_positive_point_is_unprocessable( + client: TestClient, project: str +) -> None: + """Negatives refine an answer; they cannot be the whole of a question.""" + answer = ask(client, project=project, asset=str(uuid4()), connection=str(uuid4()), positive=[]) + assert answer.status_code == 422 + + +def test_a_request_naming_no_geometry_kinds_is_unprocessable( + client: TestClient, project: str +) -> None: + answer = ask(client, project=project, asset=str(uuid4()), connection=str(uuid4()), allowed=[]) + assert answer.status_code == 422 + + +# --- the answer --------------------------------------------------------------- + + +def test_a_click_comes_back_as_a_polygon_with_its_confidence_and_model( + client: TestClient, runner: InlineDispatcher, project: str, tmp_path: Path, answering: list[Any] +) -> None: + connection = a_connection(client) + asset = an_asset(client, runner, project, tmp_path) + + body = ask(client, project=project, asset=asset, connection=connection).json() + + assert body["model_ref"] == "some/segmenter@abc123" + assert body["region"]["confidence"] == pytest.approx(0.82) + assert body["region"]["geometry"]["type"] == "polygon" + assert len(body["region"]["geometry"]["points"]) == 4 + + +def test_a_box_only_class_is_offered_the_outlines_extent( + client: TestClient, runner: InlineDispatcher, project: str, tmp_path: Path, answering: list[Any] +) -> None: + """D3's fallback over HTTP, and the test the PR body names for the geometry mutation. + + Stop respecting ``allowed_geometries`` — return the polygon regardless — and + this goes red while every other assertion in the file stays green, because a + polygon is a perfectly valid answer to every *other* question asked here. + """ + connection = a_connection(client) + asset = an_asset(client, runner, project, tmp_path) + + body = ask(client, project=project, asset=asset, connection=connection, allowed=["bbox"]).json() + + assert body["region"]["geometry"] == { + "type": "bbox", + "x": 2.0, + "y": 3.0, + "width": 10.0, + "height": 6.0, + } + + +def test_a_tag_only_class_is_offered_nothing_rather_than_a_shape_it_cannot_hold( + client: TestClient, runner: InlineDispatcher, project: str, tmp_path: Path, answering: list[Any] +) -> None: + """D3's third case. Not an error — the gesture simply has nothing to propose.""" + connection = a_connection(client) + asset = an_asset(client, runner, project, tmp_path) + + body = ask( + client, + project=project, + asset=asset, + connection=connection, + allowed=["classification_tag"], + ).json() + + assert body["region"] is None + assert body["model_ref"] == "some/segmenter@abc123", "still says who was asked" + + +def test_negative_points_travel_to_the_provider( + client: TestClient, runner: InlineDispatcher, project: str, tmp_path: Path, answering: list[Any] +) -> None: + connection = a_connection(client) + asset = an_asset(client, runner, project, tmp_path) + + ask( + client, + project=project, + asset=asset, + connection=connection, + positive=[{"x": 1.0, "y": 2.0}], + negative=[{"x": 3.0, "y": 4.0}], + ) + + prompt = answering[0].prompt + assert prompt.positive == ((1.0, 2.0),) + assert prompt.negative == ((3.0, 4.0),) + + +def test_the_provider_is_handed_bytes_rather_than_a_path( + client: TestClient, runner: InlineDispatcher, project: str, tmp_path: Path, answering: list[Any] +) -> None: + """The port's dual test, observed end to end: a hosted provider shares no filesystem.""" + connection = a_connection(client) + asset = an_asset(client, runner, project, tmp_path) + + ask(client, project=project, asset=asset, connection=connection) + + (target,) = answering[0].targets + assert isinstance(target.content, bytes) and target.content[:4] == b"\x89PNG" + assert target.media_type == "image/png" + + +def test_nothing_is_written_by_asking( + client: TestClient, runner: InlineDispatcher, project: str, tmp_path: Path, answering: list[Any] +) -> None: + """A suggestion is a proposal. Accepting it is a separate, ordinary annotation write.""" + connection = a_connection(client) + asset = an_asset(client, runner, project, tmp_path) + before = client.get(f"/projects/{project}/assets/{asset}").json() + + ask(client, project=project, asset=asset, connection=connection) + + assert client.get(f"/projects/{project}/assets/{asset}").json() == before