diff --git a/internal/cli/data.go b/internal/cli/data.go index dad1c2d..310ecb2 100644 --- a/internal/cli/data.go +++ b/internal/cli/data.go @@ -502,6 +502,17 @@ other collaborators train against it without ever seeing the raw files.`)) // synthesized spec carries the right fields before validation. switch { case push.IsTabular(a.Spec.Category): + // P3 (cli#71): a BOM'd tabular CSV is doomed in-cluster AND would + // corrupt InferSchema's own header read below — reject before + // either. The rest of the content preflight runs after the spec + // schema validation (mirroring the in-cluster order). + if perr := push.CheckTabularBOM(layout.LabelsCSV); perr != nil { + return &exitError{code: 3, err: perr} + } + if perr := push.CheckHasDataRows(layout.LabelsCSV); perr != nil { + return &exitError{code: 3, err: perr} + } + // Column schema: an explicit --schema wins; otherwise infer // INT/FLOAT/VARCHAR types from the CSV so the customer doesn't // hand-write one for the common case. @@ -615,6 +626,15 @@ other collaborators train against it without ever seeing the raw files.`)) return &exitError{code: 2, err: errors.New("synthesized spec failed schema validation; check the flag values above")} } + // P3 content preflight (backend#828, cli#69/#71/#72/#73): preview the + // ingestor's own validators locally — AFTER the spec schema validation, + // mirroring the in-cluster order (jsonschema first, then validators), + // and BEFORE any cluster work. Each check names the rule it previews; + // parity is pinned by internal/push/parity_golden_test.go. + if perr := runLocalPreflight(a, layout, errOut); perr != nil { + return perr + } + printLocalSummary(a.Printer, layout, spec) // 5. Cluster discovery — same kubeconfig path as `cluster info`. @@ -1042,3 +1062,22 @@ func destTableExists(ctx context.Context, cs kubernetes.Interface, resolved *clu } return "", "" } + +// runLocalPreflight maps push.PreflightDataset — THE shared preview +// dispatch, also exercised verbatim by the parity harness — onto the CLI's +// conventions: notes print dim to errOut, a BadFlag problem exits 2 (fix a +// flag), anything else exits 3 (fix the data). +func runLocalPreflight(a runDataIngestArgs, layout *push.LocalLayout, errOut io.Writer) error { + notes, problem := push.PreflightDataset(a.Spec, layout) + for _, n := range notes { + _, _ = fmt.Fprintln(errOut, n) + } + if problem == nil { + return nil + } + code := 3 + if problem.BadFlag { + code = 2 + } + return &exitError{code: code, err: problem.Err} +} diff --git a/internal/cli/data_test.go b/internal/cli/data_test.go index c31c696..703191e 100644 --- a/internal/cli/data_test.go +++ b/internal/cli/data_test.go @@ -3,6 +3,8 @@ package cli import ( "github.com/tracebloc/cli/internal/push" "github.com/tracebloc/cli/internal/ui" + "image" + "image/png" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" @@ -24,16 +26,25 @@ func imgcLayout(t *testing.T) string { t.Helper() root := t.TempDir() if err := os.WriteFile(filepath.Join(root, "labels.csv"), - []byte("image_id,label\n001.jpg,cat\n"), 0o644); err != nil { + []byte("image_id,label\n001.jpg,cat\n002.jpg,dog\n"), 0o644); err != nil { t.Fatalf("write labels.csv: %v", err) } imagesDir := filepath.Join(root, "images") if err := os.MkdirAll(imagesDir, 0o755); err != nil { t.Fatalf("mkdir images: %v", err) } - if err := os.WriteFile(filepath.Join(imagesDir, "001.jpg"), - make([]byte, 100), 0o644); err != nil { - t.Fatalf("write image: %v", err) + // Real decodable images with two classes: the P3 preflight decodes + // EVERY image and requires >=2 label classes, so opaque stubs would + // short-circuit any test that needs to get past preflight (a + // kubeconfig test failed vacuously on exactly that). + var buf bytes.Buffer + if err := png.Encode(&buf, image.NewRGBA(image.Rect(0, 0, 4, 4))); err != nil { + t.Fatal(err) + } + for _, name := range []string{"001.jpg", "002.jpg"} { + if err := os.WriteFile(filepath.Join(imagesDir, name), buf.Bytes(), 0o644); err != nil { + t.Fatalf("write %s: %v", name, err) + } } return root } diff --git a/internal/push/parity_golden_test.go b/internal/push/parity_golden_test.go new file mode 100644 index 0000000..fda5a72 --- /dev/null +++ b/internal/push/parity_golden_test.go @@ -0,0 +1,126 @@ +package push + +import ( + "encoding/json" + "os" + "path/filepath" + "sort" + "testing" +) + +// The validator-parity harness (backend#828 P3). Two assertions per case: +// +// 1. the Go preflight's verdict matches the manifest's cli_verdict — +// pins the CLI side; +// 2. the COMMITTED goldens (generated from the real data-ingestors +// validators by scripts/gen-validator-goldens.py) match the +// manifest's ingestor_verdict — so when the ingestor's rules change, +// regenerating the goldens fails this test until the manifest (and, +// where needed, the Go preview) is consciously updated. +// +// Deliberate divergences (the CLI previewing read-/transfer-time failures +// the ingestor's preflight can't see) are explicit in the manifest, never +// silent. + +type parityCase struct { + Name string `json:"name"` + Category string `json:"category"` + CSV string `json:"csv"` + LabelColumn string `json:"label_column"` + Extension string `json:"extension"` + TargetSize []int `json:"target_size"` + CLIVerdict string `json:"cli_verdict"` + IngestorVerdict string `json:"ingestor_verdict"` + Note string `json:"note"` +} + +func TestValidatorParity(t *testing.T) { + var manifest struct { + Cases []parityCase `json:"cases"` + } + mustLoad(t, filepath.Join("testdata", "parity", "cases.json"), &manifest) + + var goldens struct { + Verdicts map[string]struct { + Verdict string `json:"verdict"` + Errors []string `json:"errors"` + } `json:"verdicts"` + } + mustLoad(t, filepath.Join("testdata", "parity", "goldens.json"), &goldens) + + for _, c := range manifest.Cases { + t.Run(c.Name, func(t *testing.T) { + golden, ok := goldens.Verdicts[c.Name] + if !ok { + t.Fatalf("no golden for %s — run scripts/gen-validator-goldens.py", c.Name) + } + if golden.Verdict != c.IngestorVerdict { + t.Errorf("the REAL ingestor validators say %q but the manifest expects %q — "+ + "the ingestor's rules changed; update cases.json (and the Go preview if needed). "+ + "Golden errors: %v", golden.Verdict, c.IngestorVerdict, golden.Errors) + } + got := runGoPreflight(t, c) + if got != c.CLIVerdict { + t.Errorf("Go preflight = %q, manifest expects %q (note: %s)", got, c.CLIVerdict, c.Note) + } + }) + } +} + +// runGoPreflight runs THE production dispatch (push.PreflightDataset) over +// the case — the same code path runDataIngest executes, so a check deleted +// or rewired in production fails parity here. +func runGoPreflight(t *testing.T, c parityCase) string { + t.Helper() + dir := filepath.Join("testdata", "parity", "cases", c.Name) + layout := &LocalLayout{ + Root: dir, + LabelsCSV: filepath.Join(dir, c.CSV), + Images: listImages(t, dir), + } + spec := SpecArgs{ + Category: c.Category, + LabelColumn: c.LabelColumn, + Extension: c.Extension, + TargetSize: c.TargetSize, + } + if IsTabular(c.Category) { + // Mirror runDataIngest: the schema is inferred from the CSV before + // the preflight runs (the schema-columns preview needs it). + if sch, _, _, err := InferSchema(layout.LabelsCSV); err == nil { + spec.Schema = sch + } + } + _, problem := PreflightDataset(spec, layout) + if problem != nil { + return "reject" + } + return "accept" +} + +func listImages(t *testing.T, dir string) []string { + t.Helper() + entries, err := os.ReadDir(filepath.Join(dir, "images")) + if err != nil { + return nil // tabular/text cases have no images/ + } + var out []string + for _, e := range entries { + if !e.IsDir() { + out = append(out, filepath.Join(dir, "images", e.Name())) + } + } + sort.Strings(out) + return out +} + +func mustLoad(t *testing.T, path string, into any) { + t.Helper() + b, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if err := json.Unmarshal(b, into); err != nil { + t.Fatalf("%s: %v", path, err) + } +} diff --git a/internal/push/preflight.go b/internal/push/preflight.go new file mode 100644 index 0000000..6d19d78 --- /dev/null +++ b/internal/push/preflight.go @@ -0,0 +1,673 @@ +// Package push — local preflight previews of the in-cluster ingestor's +// validators (backend#828 P3; closes cli#69/#71/#72/#73). +// +// THE CONTRACT: every check in this file previews a NAMED rule in +// tracebloc/data-ingestors, so "preflight passed" means "the ingestor's +// validation will pass" — the customer finds out BEFORE the upload, not +// after. The CLI never invents rules of its own here: stricter-than-ingestor +// checks would reject datasets the cluster accepts, looser ones burn +// uploads. Parity is pinned by internal/push/parity_golden_test.go against +// goldens generated from the real Python validators +// (scripts/gen-validator-goldens.py); regenerate them whenever the +// ingestor's rules change. +package push + +import ( + "bufio" + "bytes" + "encoding/csv" + "errors" + "fmt" + "image" + "io" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "unicode/utf8" +) + +// utf8BOM is the byte-order mark Excel's "CSV UTF-8" export prepends. +var utf8BOM = []byte{0xEF, 0xBB, 0xBF} + +// HasBOM reports whether the file starts with a UTF-8 BOM. +func HasBOM(path string) (bool, error) { + f, err := os.Open(path) + if err != nil { + return false, err + } + defer f.Close() + head := make([]byte, 3) + n, err := io.ReadFull(f, head) + if err != nil && n < 3 { + return false, nil // shorter than a BOM — trivially no BOM + } + return bytes.Equal(head, utf8BOM), nil +} + +// ReadCSVHeader returns the first record of the CSV with a UTF-8 BOM (if +// any) stripped and surrounding whitespace trimmed from each name. +// +// BOM parity (cli#71): the ingestor reads CSVs with pandas, which strips +// the BOM even under encoding="utf-8" — so for the pandas-backed checks +// (label column, row counts) a BOM'd file behaves as if it had none, and +// this reader must match or the CLI would falsely reject what the cluster +// accepts. The one in-cluster path that does NOT strip it is the tabular +// schema probe — see CheckTabularBOM. +func ReadCSVHeader(path string) ([]string, error) { + f, err := os.Open(path) + if err != nil { + return nil, err + } + defer f.Close() + br := bufio.NewReader(f) + if head, _ := br.Peek(3); bytes.Equal(head, utf8BOM) { + _, _ = br.Discard(3) + } + r := csv.NewReader(br) + header, err := r.Read() + if err != nil { + if errors.Is(err, io.EOF) { + return nil, fmt.Errorf("%s is empty — no header row", filepath.Base(path)) + } + return nil, fmt.Errorf("reading %s header: %w", filepath.Base(path), err) + } + for i := range header { + header[i] = strings.TrimSpace(header[i]) + } + return header, nil +} + +// CheckTabularBOM previews the in-cluster tabular schema probe +// (data-ingestors data_validator.py _validate_csv_streaming): unlike the +// pandas paths, that probe reads the header with Python's stdlib csv over +// plain utf-8, so a BOM glues U+FEFF onto the first column name and the +// schema check FALSELY rejects the file after the full upload +// ("Schema columns not present in CSV: "). Until that +// ingestor bug is fixed and deployed, a BOM'd tabular CSV is doomed +// in-cluster — reject it here, before the upload, with the actual fix. +func CheckTabularBOM(path string) error { + bom, err := HasBOM(path) + if err != nil { + return fmt.Errorf("checking %s for a byte-order mark: %w", filepath.Base(path), err) + } + if !bom { + return nil + } + return fmt.Errorf( + "%s starts with a UTF-8 byte-order mark (Excel's \"CSV UTF-8\" export adds it), "+ + "which the cluster's schema check can't read — the ingestion would fail after "+ + "uploading everything. Re-save it without the mark (in Excel choose plain \"CSV\"; "+ + "or: tail -c +4 %s > fixed.csv) and re-run.", + filepath.Base(path), filepath.Base(path)) +} + +// CheckLabelColumn previews the ingestor's LabelColumnValidator +// (label_column_validator.py): the configured label column must exist in +// the CSV header — matched exactly first, then case-insensitively with +// surrounding whitespace stripped (the ingestor's _match_column rule; it +// must stay this loose or the CLI would reject datasets the cluster +// accepts). +func CheckLabelColumn(header []string, labelColumn, csvName string) error { + for _, c := range header { + if c == labelColumn { + return nil + } + } + want := strings.ToLower(strings.TrimSpace(labelColumn)) + for _, c := range header { + if strings.ToLower(strings.TrimSpace(c)) == want { + return nil + } + } + return fmt.Errorf( + "label column %q isn't in %s's header (columns: %s). Pass --label-column with one of "+ + "the existing columns, or add a %q column to the CSV.", + labelColumn, csvName, strings.Join(header, ", "), labelColumn) +} + +// CheckDuplicateHeaders previews the ingestor's duplicate-header probes +// (data_validator.py preflight + csv_ingestor.py read time): duplicate +// column names — compared stripped but case-SENSITIVE, exactly as the +// ingestor compares them — are rejected there, and the read-time copy +// fires after table creation, leaving an orphaned empty table. Catch it +// locally instead. (InferSchema's map keying would also silently collapse +// the duplicate — cli#73a.) +func CheckDuplicateHeaders(header []string, csvName string) error { + seen := make(map[string]bool, len(header)) + var dups []string + for _, c := range header { + if seen[c] { + dups = append(dups, c) + } + seen[c] = true + } + if len(dups) == 0 { + return nil + } + sort.Strings(dups) + return fmt.Errorf( + "%s has duplicate column name(s): %s. Each column must be unique — the cluster "+ + "rejects duplicates, and the schema would map onto the wrong column. Rename them and re-run.", + csvName, strings.Join(dups, ", ")) +} + +// CheckHasDataRows previews the ingestor's IngestableRecordsValidator +// (ingestable_records_validator.py _check_has_rows, run for every +// category): a header-only CSV has zero ingestable records and is +// rejected in-cluster before any table is created. +func CheckHasDataRows(path string) error { + f, err := os.Open(path) + if err != nil { + return fmt.Errorf("reading %s: %w", filepath.Base(path), err) + } + defer f.Close() + br := bufio.NewReader(f) + if head, _ := br.Peek(3); bytes.Equal(head, utf8BOM) { + _, _ = br.Discard(3) + } + r := csv.NewReader(br) + r.FieldsPerRecord = -1 // row-shape problems are someone else's diagnostic + if _, err := r.Read(); err != nil { + if errors.Is(err, io.EOF) { + return fmt.Errorf("%s is empty — add a header and at least one data row, then re-run", filepath.Base(path)) + } + return fmt.Errorf("reading %s: %w", filepath.Base(path), err) + } + if _, err := r.Read(); err != nil { + if errors.Is(err, io.EOF) { + return fmt.Errorf( + "%s has a header but no data rows (0 ingestable records). Add at least one data row and re-run.", + filepath.Base(path)) + } + return fmt.Errorf("reading %s: %w", filepath.Base(path), err) + } + return nil +} + +// ValidateImages previews the ingestor's ImageResolutionValidator +// (image_validator.py): it opens EVERY image (header-only decode — cheap) +// and rejects zero-byte files, undecodable files, and any image whose +// resolution differs from the expected size (exact equality, zero +// tolerance — the ingestor validates, it does not resize). Previously the +// CLI decoded only the first image, so a single odd-sized or corrupt file +// failed in-cluster after the full upload (cli#72b/c). +// +// expectedW/expectedH of 0 skips the resolution comparison (the caller +// couldn't establish a target size — the ingestor would then auto-detect +// from its first file, which the CLI's detection already mirrors). +func ValidateImages(images []string, expectedW, expectedH int) error { + const maxListed = 5 + var broken, mismatched []string + for _, path := range images { + name := filepath.Base(path) + f, err := os.Open(path) + if err != nil { + broken = append(broken, fmt.Sprintf("%s (unreadable: %v)", name, err)) + continue + } + cfg, _, err := image.DecodeConfig(f) + _ = f.Close() + if err != nil { + if st, serr := os.Stat(path); serr == nil && st.Size() == 0 { + broken = append(broken, name+" (empty file, 0 bytes)") + } else { + broken = append(broken, name+" (not a valid image — corrupt or unsupported format)") + } + continue + } + if expectedW > 0 && expectedH > 0 && (cfg.Width != expectedW || cfg.Height != expectedH) { + mismatched = append(mismatched, + fmt.Sprintf("%s (%dx%d)", name, cfg.Width, cfg.Height)) + } + } + if len(broken) > 0 { + return fmt.Errorf( + "%d image(s) can't be ingested: %s. The cluster rejects these after the upload — "+ + "fix or remove them and re-run.", + len(broken), TruncateList(broken, maxListed)) + } + if len(mismatched) > 0 { + return fmt.Errorf( + "%d image(s) don't match the %dx%d resolution: %s. The cluster validates the size, "+ + "it does not resize — make them uniform, or pass --target-size to match your data.", + len(mismatched), expectedW, expectedH, TruncateList(mismatched, maxListed)) + } + return nil +} + +// CrossCheckLabels previews the transfer-time fate of each labels.csv row +// for image_classification (file_transfer.py _find_src): a row whose image +// file doesn't exist under images/ is dropped as a failed record — the run +// then "completes with failures" (exit 9) after the full upload. The +// filename column may omit the extension; the ingestor appends the +// dataset's extension in that case, and this check mirrors that. Files +// with no CSV row are NOT an error (the ingestor never checks that +// direction for image_classification) — the caller may surface them as a +// note. +// +// filenameColumn is the CSV's first column (the ingestor reads filenames +// positionally from the id column of labels.csv). +func CrossCheckLabels(csvPath string, images []string, extension string) (missing []string, orphans []string, err error) { + f, err := os.Open(csvPath) + if err != nil { + return nil, nil, fmt.Errorf("reading %s: %w", filepath.Base(csvPath), err) + } + defer f.Close() + br := bufio.NewReader(f) + if head, _ := br.Peek(3); bytes.Equal(head, utf8BOM) { + _, _ = br.Discard(3) + } + r := csv.NewReader(br) + r.FieldsPerRecord = -1 + + present := make(map[string]bool, len(images)) + for _, img := range images { + present[filepath.Base(img)] = true + } + referenced := make(map[string]bool) + + if _, err := r.Read(); err != nil { // header + if errors.Is(err, io.EOF) { + return nil, nil, nil // emptiness is CheckHasDataRows' diagnostic + } + return nil, nil, fmt.Errorf("reading %s: %w", filepath.Base(csvPath), err) + } + for { + rec, err := r.Read() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + return nil, nil, fmt.Errorf("reading %s: %w", filepath.Base(csvPath), err) + } + if len(rec) == 0 { + continue + } + name := strings.TrimSpace(rec[0]) + if name == "" { + continue + } + // Mirror the ingestor's extension handling (_has_extension): the + // dataset extension is appended unless the name already ends in a + // KNOWN media extension — a dotted stem like "img.2024" is not an + // extension to the ingestor, and must not be one here either. + if !hasKnownExtension(name) && extension != "" { + name += extension + } + referenced[name] = true + if !present[name] { + missing = append(missing, name) + } + } + for name := range present { + if !referenced[name] { + orphans = append(orphans, name) + } + } + sort.Strings(missing) + sort.Strings(orphans) + return missing, orphans, nil +} + +// CheckAnnotationPairing previews the ingestor's FilePairingValidator +// (file_pairing_validator.py) for object_detection: every image must have +// an annotation with the same filename stem and vice versa — a mismatch in +// either direction fails in-cluster after the upload. +func CheckAnnotationPairing(images, annotations []string) error { + const maxListed = 5 + stems := func(paths []string) map[string]bool { + m := make(map[string]bool, len(paths)) + for _, p := range paths { + base := filepath.Base(p) + m[strings.TrimSuffix(base, filepath.Ext(base))] = true + } + return m + } + imgStems, annStems := stems(images), stems(annotations) + var noAnn, noImg []string + for s := range imgStems { + if !annStems[s] { + noAnn = append(noAnn, s) + } + } + for s := range annStems { + if !imgStems[s] { + noImg = append(noImg, s) + } + } + if len(noAnn) == 0 && len(noImg) == 0 { + return nil + } + sort.Strings(noAnn) + sort.Strings(noImg) + var parts []string + if len(noAnn) > 0 { + parts = append(parts, fmt.Sprintf("%d image(s) without an annotation (%s)", + len(noAnn), TruncateList(noAnn, maxListed))) + } + if len(noImg) > 0 { + parts = append(parts, fmt.Sprintf("%d annotation(s) without an image (%s)", + len(noImg), TruncateList(noImg, maxListed))) + } + return fmt.Errorf( + "images/ and annotations/ don't pair up: %s. Every image needs a same-named .xml "+ + "annotation (and vice versa) — the cluster rejects mismatches after the upload.", + strings.Join(parts, "; ")) +} + +// TruncateList joins up to max items, appending "… and N more" past that. +func TruncateList(items []string, max int) string { + if len(items) <= max { + return strings.Join(items, ", ") + } + return fmt.Sprintf("%s, … and %d more", strings.Join(items[:max], ", "), len(items)-max) +} + +// CheckLabelDiversity previews the ingestor's LabelDiversityValidator +// (label_diversity_validator.py, wired for every classification category): +// a classification dataset needs at least 2 distinct label values +// (whitespace-stripped) — a single class can't train a classifier, and the +// in-cluster rejection otherwise lands after the full upload. Mirrors the +// validator's benign-skip when the label column isn't found (that's +// CheckLabelColumn's diagnostic, not this one's). +// tabularSchema is non-nil for tabular_classification: the label is a +// SCHEMA-TYPED column there, so pandas drops NA-sentinel values to NaN +// (not classes) and numeric inference collapses "1"/"1.0" — the preview +// mirrors both. For image/text classification the label column is read +// untyped with keep_default_na=False, so even an empty string is a real +// class and every distinct trimmed string counts. +func CheckLabelDiversity(csvPath, labelColumn string, tabularSchema bool) error { + f, err := os.Open(csvPath) + if err != nil { + return nil // unreadable file is another check's diagnostic + } + defer f.Close() + br := bufio.NewReader(f) + if head, _ := br.Peek(3); bytes.Equal(head, utf8BOM) { + _, _ = br.Discard(3) + } + r := csv.NewReader(br) + r.FieldsPerRecord = -1 + header, err := r.Read() + if err != nil { + return nil + } + col := -1 + for i, c := range header { + if strings.TrimSpace(c) == labelColumn { + col = i + break + } + } + if col == -1 { + want := strings.ToLower(strings.TrimSpace(labelColumn)) + for i, c := range header { + if strings.ToLower(strings.TrimSpace(c)) == want { + col = i + break + } + } + } + if col == -1 { + return nil // benign-skip, like the ingestor + } + distinct := map[string]bool{} + for { + rec, err := r.Read() + if errors.Is(err, io.EOF) { + break + } + if err != nil || len(rec) <= col { + continue + } + v := strings.TrimSpace(rec[col]) + if tabularSchema { + if _, isNA := naSentinels[v]; isNA { + continue + } + // Numeric inference collapses "1" and "1.0" into one value + // in-cluster; normalize the same way before counting. + if f, err := strconv.ParseFloat(v, 64); err == nil { + v = strconv.FormatFloat(f, 'g', -1, 64) + } + } + distinct[v] = true + if len(distinct) >= 2 { + return nil + } + } + if len(distinct) >= 2 { + return nil + } + return fmt.Errorf( + "the label column %q has %d distinct value(s) — a classification dataset needs at "+ + "least 2 classes. The cluster rejects this after the upload; check the labels and re-run.", + labelColumn, len(distinct)) +} + +// knownMediaExtensions mirrors the ingestor's FileExtension.get_all_extensions +// (utils/constants.py): the ONLY suffixes file_transfer._has_extension treats +// as extensions. Anything else — img.2024, photo.v2 — gets the dataset +// extension appended by the ingestor, and CrossCheckLabels must mirror that +// or it rejects dotted stems the cluster resolves fine. +var knownMediaExtensions = map[string]struct{}{ + ".jpeg": {}, ".jpg": {}, ".png": {}, ".xml": {}, ".txt": {}, ".text": {}, +} + +// hasKnownExtension previews file_transfer._has_extension: true only when the +// name's final dot-suffix is one of the ingestor's known extensions +// (case-insensitive). +func hasKnownExtension(name string) bool { + _, ok := knownMediaExtensions[strings.ToLower(filepath.Ext(name))] + return ok +} + +// CheckCSVEncoding previews the ingestor's preflight.check_csv_encoding — +// the FIRST gate validate_data runs in-cluster: the CSV must be valid UTF-8 +// and free of NUL bytes, or the whole run aborts (after the upload). +func CheckCSVEncoding(path string) error { + f, err := os.Open(path) + if err != nil { + return fmt.Errorf("reading %s: %w", filepath.Base(path), err) + } + defer f.Close() + data, err := io.ReadAll(io.LimitReader(f, MaxSingleFileBytes+1)) + if err != nil { + return fmt.Errorf("reading %s: %w", filepath.Base(path), err) + } + if bytes.IndexByte(data, 0x00) >= 0 { + return fmt.Errorf( + "%s contains a NUL byte — the file is corrupt or not really a CSV. The cluster "+ + "rejects it after the upload; re-export the file and re-run.", + filepath.Base(path)) + } + if !utf8.Valid(data) { + return fmt.Errorf( + "%s isn't valid UTF-8 (likely a Latin-1/Windows-1252 export). The cluster rejects "+ + "non-UTF-8 CSVs after the upload — re-save it as UTF-8 and re-run.", + filepath.Base(path)) + } + return nil +} + +// naSentinels mirrors the ingestor's coercion.NA_SENTINELS: values pandas +// drops to NaN for SCHEMA-TYPED columns (tabular labels), and therefore +// values the in-cluster LabelDiversityValidator does not count as classes. +var naSentinels = map[string]struct{}{ + "": {}, "NA": {}, "N/A": {}, "n/a": {}, "NULL": {}, "null": {}, + "None": {}, "none": {}, "NaN": {}, "nan": {}, "": {}, "#N/A": {}, +} + +// CheckSchemaColumns previews DataValidator's missing-schema-column probe +// ("Schema columns not present in CSV: …"): every schema column must appear +// in the header, compared stripped but case-SENSITIVE — exactly the probe's +// set difference. +func CheckSchemaColumns(header []string, schema map[string]string, csvName string) error { + present := make(map[string]bool, len(header)) + for _, c := range header { + present[strings.TrimSpace(c)] = true + } + var missing []string + for col := range schema { + if !present[strings.TrimSpace(col)] { + missing = append(missing, col) + } + } + if len(missing) == 0 { + return nil + } + sort.Strings(missing) + return fmt.Errorf( + "--schema names column(s) that aren't in %s: %s. The cluster rejects this after the "+ + "upload — fix the schema or the CSV header, then re-run.", + csvName, strings.Join(missing, ", ")) +} + +// PreflightProblem is a preflight rejection. BadFlag marks problems whose +// fix is a flag value (the CLI maps those to exit 2); everything else is a +// data problem (exit 3). +type PreflightProblem struct { + Err error + BadFlag bool +} + +// PreflightDataset is THE preflight dispatch — the single place that decides +// which previews run for which category, shared verbatim by the CLI +// (runLocalPreflight) and the parity harness (parity_golden_test.go), so the +// two cannot drift: deleting or rewiring a check here fails the parity test. +// Returned notes are advisory (the CLI prints them dim); a non-nil problem +// is the rejection. +// +// Ordering mirrors the in-cluster pipeline: encoding gate first (the +// ingestor's check_csv_encoding runs before any validator), then the +// validator previews. +func PreflightDataset(spec SpecArgs, layout *LocalLayout) (notes []string, problem *PreflightProblem) { + dataProblem := func(err error) *PreflightProblem { + if err == nil { + return nil + } + return &PreflightProblem{Err: err} + } + + switch { + case IsTabular(spec.Category): + if err := CheckTabularBOM(layout.LabelsCSV); err != nil { + return nil, dataProblem(err) + } + if err := CheckCSVEncoding(layout.LabelsCSV); err != nil { + return nil, dataProblem(err) + } + header, err := ReadCSVHeader(layout.LabelsCSV) + if err != nil { + return nil, dataProblem(err) + } + if err := CheckDuplicateHeaders(header, "the data CSV"); err != nil { + return nil, dataProblem(err) + } + if err := CheckHasDataRows(layout.LabelsCSV); err != nil { + return nil, dataProblem(err) + } + if err := CheckSchemaColumns(header, spec.Schema, "the data CSV"); err != nil { + return nil, dataProblem(err) + } + // A bogus --label-column otherwise fails in-cluster only at READ + // time — after the table was created — leaving an orphaned table. + if err := CheckLabelColumn(header, spec.LabelColumn, "the data CSV"); err != nil { + return nil, &PreflightProblem{Err: err, BadFlag: true} + } + if spec.Category == "tabular_classification" { + if err := CheckLabelDiversity(layout.LabelsCSV, spec.LabelColumn, true); err != nil { + return nil, dataProblem(err) + } + } + + case IsImage(spec.Category): + if err := CheckCSVEncoding(layout.LabelsCSV); err != nil { + return nil, dataProblem(err) + } + // Every image, header-only decode (cheap). Previously only the + // first image was ever opened, so one corrupt or odd-sized file + // failed in-cluster after the full upload (cli#72). + expW, expH := 0, 0 + if len(spec.TargetSize) == 2 { + expW, expH = spec.TargetSize[0], spec.TargetSize[1] + } + if err := ValidateImages(layout.Images, expW, expH); err != nil { + return nil, dataProblem(err) + } + if err := CheckHasDataRows(layout.LabelsCSV); err != nil { + return nil, dataProblem(err) + } + header, err := ReadCSVHeader(layout.LabelsCSV) + if err != nil { + return nil, dataProblem(err) + } + if err := CheckDuplicateHeaders(header, "labels.csv"); err != nil { + return nil, dataProblem(err) + } + // LabelDiversityValidator runs in-cluster for the WHOLE image + // family (is_classification covers object_detection + keypoint + // too); it benign-skips when no label column resolves, and so + // does the preview. + if err := CheckLabelDiversity(layout.LabelsCSV, spec.LabelColumn, false); err != nil { + return nil, dataProblem(err) + } + switch spec.Category { + case "image_classification": + if err := CheckLabelColumn(header, spec.LabelColumn, "labels.csv"); err != nil { + return nil, &PreflightProblem{Err: err, BadFlag: true} + } + // A row whose image is missing becomes a failed record + // in-cluster ("completed with failures", exit 9) — after the + // upload. Extra files are only a note (the ingestor never + // checks that direction). + missing, orphanFiles, err := CrossCheckLabels(layout.LabelsCSV, layout.Images, spec.Extension) + if err != nil { + return nil, dataProblem(err) + } + if len(missing) > 0 { + return nil, dataProblem(fmt.Errorf( + "%d labels.csv row(s) reference images that aren't in images/: %s. Those records "+ + "would fail after the upload — fix the rows or add the files, then re-run.", + len(missing), TruncateList(missing, 5))) + } + if len(orphanFiles) > 0 { + notes = append(notes, fmt.Sprintf( + "Note: %d file(s) in images/ have no labels.csv row and won't be part of the dataset: %s", + len(orphanFiles), TruncateList(orphanFiles, 5))) + } + case "object_detection": + // images↔annotations stem pairing (FilePairingValidator preview). + if err := CheckAnnotationPairing(layout.Images, layout.Sidecars["annotations"]); err != nil { + return nil, dataProblem(err) + } + } + + default: + // Text family. + if err := CheckCSVEncoding(layout.LabelsCSV); err != nil { + return nil, dataProblem(err) + } + if err := CheckHasDataRows(layout.LabelsCSV); err != nil { + return nil, dataProblem(err) + } + header, err := ReadCSVHeader(layout.LabelsCSV) + if err != nil { + return nil, dataProblem(err) + } + if err := CheckDuplicateHeaders(header, "labels.csv"); err != nil { + return nil, dataProblem(err) + } + if spec.Category == "text_classification" { + if err := CheckLabelColumn(header, spec.LabelColumn, "labels.csv"); err != nil { + return nil, &PreflightProblem{Err: err, BadFlag: true} + } + if err := CheckLabelDiversity(layout.LabelsCSV, spec.LabelColumn, false); err != nil { + return nil, dataProblem(err) + } + } + } + return notes, nil +} diff --git a/internal/push/preflight_test.go b/internal/push/preflight_test.go new file mode 100644 index 0000000..1baa772 --- /dev/null +++ b/internal/push/preflight_test.go @@ -0,0 +1,218 @@ +package push + +import ( + "image" + "image/png" + "os" + "path/filepath" + "strings" + "testing" +) + +func writeTmp(t *testing.T, name string, body []byte) string { + t.Helper() + p := filepath.Join(t.TempDir(), name) + if err := os.WriteFile(p, body, 0o644); err != nil { + t.Fatal(err) + } + return p +} + +var bom = []byte{0xEF, 0xBB, 0xBF} + +func TestReadCSVHeader_StripsBOMAndTrims(t *testing.T) { + // Parity (cli#71): pandas strips the BOM in-cluster, so the local + // reader must too — otherwise the CLI would reject label columns the + // cluster accepts. + p := writeTmp(t, "labels.csv", append(bom, []byte(" filename , label \nx.jpg,cat\n")...)) + h, err := ReadCSVHeader(p) + if err != nil { + t.Fatal(err) + } + if h[0] != "filename" || h[1] != "label" { + t.Errorf("header = %v, want BOM-stripped + trimmed [filename label]", h) + } +} + +func TestCheckTabularBOM(t *testing.T) { + // Parity (cli#71): the in-cluster tabular schema probe does NOT strip + // the BOM and falsely rejects — the CLI must reject locally, before + // the upload, with the actual fix. + withBOM := writeTmp(t, "d.csv", append(bom, []byte("age,income\n1,2\n")...)) + if err := CheckTabularBOM(withBOM); err == nil { + t.Fatal("BOM'd tabular CSV must be rejected (in-cluster schema probe would falsely reject it post-upload)") + } else if !strings.Contains(err.Error(), "byte-order mark") || !strings.Contains(err.Error(), "Re-save") { + t.Errorf("BOM error must explain + remediate, got: %v", err) + } + clean := writeTmp(t, "c.csv", []byte("age,income\n1,2\n")) + if err := CheckTabularBOM(clean); err != nil { + t.Errorf("clean CSV rejected: %v", err) + } +} + +func TestCheckLabelColumn_MatchesLikeTheIngestor(t *testing.T) { + // Parity (cli#69): exact first, then case-insensitive + trimmed — + // the ingestor's _match_column rule. Stricter matching would reject + // datasets the cluster accepts. + header := []string{"filename", " Label "} + if err := CheckLabelColumn(header, "label", "labels.csv"); err != nil { + t.Errorf("case-insensitive+trimmed match must pass (ingestor accepts it): %v", err) + } + if err := CheckLabelColumn(header, "target", "labels.csv"); err == nil { + t.Fatal("absent label column must be rejected") + } else { + for _, want := range []string{`"target"`, "filename", "--label-column"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("error should mention %q, got: %v", want, err) + } + } + } +} + +func TestCheckDuplicateHeaders_CaseSensitiveLikeTheIngestor(t *testing.T) { + // Parity (cli#73a): the ingestor compares stripped but case-SENSITIVE. + if err := CheckDuplicateHeaders([]string{"a", "A"}, "the data CSV"); err != nil { + t.Errorf("'a' vs 'A' are NOT duplicates to the ingestor: %v", err) + } + err := CheckDuplicateHeaders([]string{"age", "income", "age"}, "the data CSV") + if err == nil { + t.Fatal("duplicate headers must be rejected") + } + if !strings.Contains(err.Error(), "age") || !strings.Contains(err.Error(), "Rename") { + t.Errorf("dup error must name the column + remediate: %v", err) + } +} + +func TestCheckHasDataRows(t *testing.T) { + ok := writeTmp(t, "ok.csv", []byte("a,b\n1,2\n")) + if err := CheckHasDataRows(ok); err != nil { + t.Errorf("CSV with rows rejected: %v", err) + } + headerOnly := writeTmp(t, "h.csv", []byte("a,b\n")) + if err := CheckHasDataRows(headerOnly); err == nil { + t.Fatal("header-only CSV must be rejected (cli#73b — 0 ingestable records)") + } else if !strings.Contains(err.Error(), "no data rows") { + t.Errorf("unexpected message: %v", err) + } + empty := writeTmp(t, "e.csv", nil) + if err := CheckHasDataRows(empty); err == nil { + t.Fatal("empty CSV must be rejected") + } +} + +func pngBytes(t *testing.T, w, h int) []byte { + t.Helper() + var sb strings.Builder + if err := png.Encode(&nopWriter{&sb}, image.NewRGBA(image.Rect(0, 0, w, h))); err != nil { + t.Fatal(err) + } + return []byte(sb.String()) +} + +type nopWriter struct{ b *strings.Builder } + +func (w *nopWriter) Write(p []byte) (int, error) { return w.b.Write(p) } + +func TestValidateImages(t *testing.T) { + dir := t.TempDir() + write := func(name string, body []byte) string { + p := filepath.Join(dir, name) + if err := os.WriteFile(p, body, 0o644); err != nil { + t.Fatal(err) + } + return p + } + good := write("good.png", pngBytes(t, 8, 8)) + odd := write("odd.png", pngBytes(t, 4, 4)) + zero := write("zero.png", nil) + corrupt := write("corrupt.png", []byte("not an image at all")) + + if err := ValidateImages([]string{good}, 8, 8); err != nil { + t.Errorf("valid image rejected: %v", err) + } + if err := ValidateImages([]string{good, zero}, 8, 8); err == nil { + t.Fatal("zero-byte image must be rejected (cli#72b)") + } else if !strings.Contains(err.Error(), "0 bytes") { + t.Errorf("zero-byte diagnosis missing: %v", err) + } + if err := ValidateImages([]string{good, corrupt}, 8, 8); err == nil { + t.Fatal("corrupt image must be rejected (cli#72b)") + } + if err := ValidateImages([]string{good, odd}, 8, 8); err == nil { + t.Fatal("resolution mismatch must be rejected (cli#72c — the ingestor validates, it does not resize)") + } else if !strings.Contains(err.Error(), "4x4") || !strings.Contains(err.Error(), "8x8") { + t.Errorf("mismatch error must show both sizes: %v", err) + } + // 0x0 expectation skips the resolution comparison entirely. + if err := ValidateImages([]string{good, odd}, 0, 0); err != nil { + t.Errorf("no expected size → no resolution rejection: %v", err) + } +} + +func TestCrossCheckLabels(t *testing.T) { + dir := t.TempDir() + imgs := filepath.Join(dir, "images") + if err := os.MkdirAll(imgs, 0o755); err != nil { + t.Fatal(err) + } + for _, n := range []string{"a.jpg", "b.jpg", "extra.jpg"} { + if err := os.WriteFile(filepath.Join(imgs, n), []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + } + // One row exact, one extensionless (the ingestor appends the dataset + // extension — the check must mirror that), one missing. + csvPath := filepath.Join(dir, "labels.csv") + if err := os.WriteFile(csvPath, + []byte("image_id,label\na.jpg,cat\nb,dog\nghost.jpg,cat\n"), 0o644); err != nil { + t.Fatal(err) + } + images := []string{filepath.Join(imgs, "a.jpg"), filepath.Join(imgs, "b.jpg"), filepath.Join(imgs, "extra.jpg")} + missing, orphans, err := CrossCheckLabels(csvPath, images, ".jpg") + if err != nil { + t.Fatal(err) + } + if len(missing) != 1 || missing[0] != "ghost.jpg" { + t.Errorf("missing = %v, want [ghost.jpg] (extensionless 'b' must resolve to b.jpg)", missing) + } + if len(orphans) != 1 || orphans[0] != "extra.jpg" { + t.Errorf("orphans = %v, want [extra.jpg]", orphans) + } +} + +func TestCheckAnnotationPairing(t *testing.T) { + imgs := []string{"images/a.jpg", "images/b.jpg"} + anns := []string{"annotations/a.xml", "annotations/c.xml"} + err := CheckAnnotationPairing(imgs, anns) + if err == nil { + t.Fatal("stem mismatch must be rejected (FilePairingValidator preview)") + } + for _, want := range []string{"b", "c", "don't pair up"} { + if !strings.Contains(err.Error(), want) { + t.Errorf("pairing error should mention %q: %v", want, err) + } + } + if err := CheckAnnotationPairing(imgs, []string{"annotations/a.xml", "annotations/b.xml"}); err != nil { + t.Errorf("matched stems rejected: %v", err) + } +} + +func TestCheckLabelDiversity(t *testing.T) { + // Parity: LabelDiversityValidator — classification needs >=2 distinct + // (stripped) label values. Discovered by the parity harness's first + // run; previously an in-cluster-only, post-upload failure. + two := writeTmp(t, "two.csv", []byte("id,label\na,cat\nb, cat \nc,dog\n")) + if err := CheckLabelDiversity(two, "label", false); err != nil { + t.Errorf("2 distinct labels rejected: %v", err) + } + one := writeTmp(t, "one.csv", []byte("id,label\na,cat\nb,cat\n")) + if err := CheckLabelDiversity(one, "label", false); err == nil { + t.Fatal("single-class dataset must be rejected") + } else if !strings.Contains(err.Error(), "at least 2 classes") { + t.Errorf("unexpected message: %v", err) + } + // benign-skip when the column is absent (that's CheckLabelColumn's job) + if err := CheckLabelDiversity(one, "nope", false); err != nil { + t.Errorf("missing column must benign-skip like the ingestor: %v", err) + } +} diff --git a/internal/push/spec.go b/internal/push/spec.go index 9845277..7ba824a 100644 --- a/internal/push/spec.go +++ b/internal/push/spec.go @@ -274,16 +274,22 @@ func (a SpecArgs) buildImage(spec map[string]any, prefix string) { if a.Category == "keypoint_detection" { if len(a.TargetSize) == 2 { - // Schema documents target_size as [height, width]; TargetSize - // is stored [W, H], so swap on emit. (Bugbot, PR #22.) - spec["target_size"] = []int{a.TargetSize[1], a.TargetSize[0]} + // Emitted as [width, height] — the schema's own description + // says so ("matches PIL.Image.size and what + // ImageResolutionValidator expects"), and the validator + // compares PIL's (W,H) against this list verbatim (verified + // empirically: an 8×4 image passes [8,4], fails [4,8]). An + // earlier revision swapped to [H,W] here on a mistaken review + // note, which made EVERY non-square dataset fail in-cluster + // after the full upload. + spec["target_size"] = []int{a.TargetSize[0], a.TargetSize[1]} } if a.NumberOfKeypoints > 0 { spec["number_of_keypoints"] = a.NumberOfKeypoints } } else if len(a.TargetSize) == 2 { - // [height, width] per the schema; TargetSize is [W, H]. - fileOptions["target_size"] = []int{a.TargetSize[1], a.TargetSize[0]} + // [width, height] — same contract as the keypoint branch above. + fileOptions["target_size"] = []int{a.TargetSize[0], a.TargetSize[1]} } if len(fileOptions) > 0 { diff --git a/internal/push/spec_test.go b/internal/push/spec_test.go index 83ed86c..72d8a79 100644 --- a/internal/push/spec_test.go +++ b/internal/push/spec_test.go @@ -197,36 +197,29 @@ func TestBuild_NoTargetSize_OmitsSpecBlock(t *testing.T) { } } -// TestBuild_TargetSize_EmittedHeightWidth locks the [height, width] -// emit order (Bugbot, PR #22). TargetSize is stored [W, H] but the -// ingest.v1 schema + ingestor read target_size as [height, width]. -// Square sizes can't catch a swap, so this uses a NON-square -// resolution: [W, H] = [640, 480] must emit [480, 640]. -func TestBuild_TargetSize_EmittedHeightWidth(t *testing.T) { - // image_classification → under spec.file_options - icSpec := SpecArgs{ +// TestBuild_TargetSize_EmittedWidthHeight locks the [width, height] +// emission order: the ingest.v1 schema's own description says target_size +// "matches PIL.Image.size" (which is (W,H)) and ImageResolutionValidator +// compares that tuple against the list verbatim — verified empirically +// against the real validator (8×4 image passes [8,4], fails [4,8]). The +// parity harness pins this end-to-end with the non-square cases. +func TestBuild_TargetSize_EmittedWidthHeight(t *testing.T) { + // TargetSize is stored [W, H] = [640, 480] (a landscape image). + imgSpec := SpecArgs{ Table: "t", Category: "image_classification", Intent: "train", - LabelColumn: "label", TargetSize: []int{640, 480}, // [W, H] + LabelColumn: "label", TargetSize: []int{640, 480}, }.Build() - sb, ok := icSpec["spec"].(map[string]any) - if !ok { - t.Fatalf("image: no spec block: %#v", icSpec["spec"]) - } - fo, ok := sb["file_options"].(map[string]any) - if !ok { - t.Fatalf("image: no file_options: %#v", sb["file_options"]) - } - if ts, ok := fo["target_size"].([]int); !ok || len(ts) != 2 || ts[0] != 480 || ts[1] != 640 { - t.Errorf("image target_size = %#v, want [480 640] (height, width)", fo["target_size"]) + fo := imgSpec["spec"].(map[string]any)["file_options"].(map[string]any) + if ts, ok := fo["target_size"].([]int); !ok || len(ts) != 2 || ts[0] != 640 || ts[1] != 480 { + t.Errorf("image target_size = %#v, want [640 480] (width, height)", fo["target_size"]) } - // keypoint_detection → top-level kpSpec := SpecArgs{ Table: "t", Category: "keypoint_detection", Intent: "train", - LabelColumn: "image_label", TargetSize: []int{640, 480}, NumberOfKeypoints: 9, + LabelColumn: "label", TargetSize: []int{640, 480}, NumberOfKeypoints: 17, }.Build() - if ts, ok := kpSpec["target_size"].([]int); !ok || len(ts) != 2 || ts[0] != 480 || ts[1] != 640 { - t.Errorf("keypoint top-level target_size = %#v, want [480 640] (height, width)", kpSpec["target_size"]) + if ts, ok := kpSpec["target_size"].([]int); !ok || len(ts) != 2 || ts[0] != 640 || ts[1] != 480 { + t.Errorf("keypoint top-level target_size = %#v, want [640 480] (width, height)", kpSpec["target_size"]) } } diff --git a/internal/push/testdata/parity/cases.json b/internal/push/testdata/parity/cases.json new file mode 100644 index 0000000..fdc4e5d --- /dev/null +++ b/internal/push/testdata/parity/cases.json @@ -0,0 +1,286 @@ +{ + "_comment": "Drives BOTH sides of the validator-parity harness (backend#828 P3). cli_verdict = what the Go preflight must decide; ingestor_verdict = what the REAL Python validators decide (pinned by goldens.json, regenerated via scripts/gen-validator-goldens.py). The two may deliberately diverge only where the note explains why (e.g. failures the ingestor only surfaces at transfer time, which the CLI previews anyway).", + "cases": [ + { + "name": "tabular-ok", + "category": "tabular_classification", + "csv": "data.csv", + "label_column": "label", + "cli_verdict": "accept", + "ingestor_verdict": "accept" + }, + { + "name": "tabular-dup-header", + "category": "tabular_classification", + "csv": "data.csv", + "label_column": "label", + "cli_verdict": "reject", + "ingestor_verdict": "reject", + "note": "DataValidator preflight probe + read-time twin (cli#73a)" + }, + { + "name": "tabular-bom", + "category": "tabular_classification", + "csv": "data.csv", + "label_column": "label", + "cli_verdict": "reject", + "ingestor_verdict": "reject", + "note": "the stdlib header probe does not strip the BOM \u2014 false in-cluster rejection the CLI must preview (cli#71)" + }, + { + "name": "tabular-header-only", + "category": "tabular_classification", + "csv": "data.csv", + "label_column": "label", + "cli_verdict": "reject", + "ingestor_verdict": "reject", + "note": "IngestableRecordsValidator (cli#73b) (residual masking: a header-only CSV also has 0 label classes, so diversity rejects too \u2014 unavoidable for this shape)" + }, + { + "name": "tabular-label-missing", + "category": "tabular_classification", + "csv": "data.csv", + "label_column": "label", + "cli_verdict": "reject", + "ingestor_verdict": "accept", + "note": "DELIBERATE divergence: the ingestor's preflight validators pass \u2014 the failure only fires at READ time (full-schema check, after table creation, orphaning it). The CLI previews the read-time rule (cli#69)." + }, + { + "name": "imgc-ok", + "category": "image_classification", + "csv": "labels.csv", + "label_column": "label", + "extension": ".jpg", + "target_size": [ + 8, + 8 + ], + "cli_verdict": "accept", + "ingestor_verdict": "accept" + }, + { + "name": "imgc-bom-labels", + "category": "image_classification", + "csv": "labels.csv", + "label_column": "label", + "extension": ".jpg", + "target_size": [ + 8, + 8 + ], + "cli_verdict": "accept", + "ingestor_verdict": "accept", + "note": "pandas strips the BOM in-cluster \u2014 the CLI must NOT reject what the cluster accepts (cli#71 parity)" + }, + { + "name": "imgc-label-missing", + "category": "image_classification", + "csv": "labels.csv", + "label_column": "label", + "extension": ".jpg", + "target_size": [ + 8, + 8 + ], + "cli_verdict": "reject", + "ingestor_verdict": "reject", + "note": "LabelColumnValidator (cli#69)" + }, + { + "name": "imgc-label-case", + "category": "image_classification", + "csv": "labels.csv", + "label_column": "label", + "extension": ".jpg", + "target_size": [ + 8, + 8 + ], + "cli_verdict": "accept", + "ingestor_verdict": "accept", + "note": "the ingestor's _match_column is case-insensitive + trimmed \u2014 the CLI must be exactly as loose" + }, + { + "name": "imgc-zero-byte", + "category": "image_classification", + "csv": "labels.csv", + "label_column": "label", + "extension": ".jpg", + "target_size": [ + 8, + 8 + ], + "cli_verdict": "reject", + "ingestor_verdict": "reject", + "note": "ImageResolutionValidator: empty file (cli#72b)" + }, + { + "name": "imgc-corrupt", + "category": "image_classification", + "csv": "labels.csv", + "label_column": "label", + "extension": ".jpg", + "target_size": [ + 8, + 8 + ], + "cli_verdict": "reject", + "ingestor_verdict": "reject", + "note": "ImageResolutionValidator: not a valid image (cli#72b)" + }, + { + "name": "imgc-res-mismatch", + "category": "image_classification", + "csv": "labels.csv", + "label_column": "label", + "extension": ".jpg", + "target_size": [ + 8, + 8 + ], + "cli_verdict": "reject", + "ingestor_verdict": "reject", + "note": "ImageResolutionValidator: exact equality, no resize (cli#72c)" + }, + { + "name": "imgc-header-only", + "category": "image_classification", + "csv": "labels.csv", + "label_column": "label", + "extension": ".jpg", + "target_size": [ + 8, + 8 + ], + "cli_verdict": "reject", + "ingestor_verdict": "reject", + "note": "IngestableRecordsValidator (cli#73b) (residual masking: a header-only CSV also has 0 label classes, so diversity rejects too \u2014 unavoidable for this shape)" + }, + { + "name": "imgc-missing-file", + "category": "image_classification", + "csv": "labels.csv", + "label_column": "label", + "extension": ".jpg", + "target_size": [ + 8, + 8 + ], + "cli_verdict": "reject", + "ingestor_verdict": "accept", + "note": "DELIBERATE divergence: the ingestor's preflight passes \u2014 the row becomes a failed record at TRANSFER time ('completed with failures', exit 9, post-upload). The CLI previews the transfer fate (cli#72a)." + }, + { + "name": "imgc-label-uniform", + "category": "image_classification", + "csv": "labels.csv", + "label_column": "label", + "extension": ".jpg", + "target_size": [ + 8, + 8 + ], + "cli_verdict": "reject", + "ingestor_verdict": "reject", + "note": "LabelDiversityValidator: classification needs >=2 distinct labels \u2014 discovered BY this harness's first run (was not in any ticket)" + }, + { + "name": "imgc-nonsquare", + "category": "image_classification", + "csv": "labels.csv", + "label_column": "label", + "extension": ".jpg", + "target_size": [ + 8, + 4 + ], + "cli_verdict": "accept", + "ingestor_verdict": "accept", + "note": "non-square [W,H]=[8,4]: pins the target_size ORIENTATION end-to-end \u2014 an [H,W] swap on emit (the pre-P3 bug) flips this to reject in-cluster" + }, + { + "name": "imgc-nonsquare-swapped", + "category": "image_classification", + "csv": "labels.csv", + "label_column": "label", + "extension": ".jpg", + "target_size": [ + 4, + 8 + ], + "cli_verdict": "reject", + "ingestor_verdict": "reject", + "note": "the same 8\u00d74 images with target [4,8]: both sides must reject \u2014 proves the comparison is orientation-sensitive, not shape-normalized" + }, + { + "name": "imgc-dup-header", + "category": "image_classification", + "csv": "labels.csv", + "label_column": "label", + "extension": ".jpg", + "target_size": [ + 8, + 8 + ], + "cli_verdict": "reject", + "ingestor_verdict": "accept", + "note": "DELIBERATE divergence: image categories have no DataValidator, so the ingestor's preflight accepts dup headers \u2014 the rejection fires at READ time (after table creation, orphaning it). The CLI previews the read-time rule." + }, + { + "name": "imgc-empty-label", + "category": "image_classification", + "csv": "labels.csv", + "label_column": "label", + "extension": ".jpg", + "target_size": [ + 8, + 8 + ], + "cli_verdict": "accept", + "ingestor_verdict": "accept", + "note": "empty-string label IS a class for image categories (keep_default_na=False): the CLI must not be stricter" + }, + { + "name": "imgc-dotted-stem", + "category": "image_classification", + "csv": "labels.csv", + "label_column": "label", + "extension": ".jpg", + "target_size": [ + 8, + 8 + ], + "cli_verdict": "accept", + "ingestor_verdict": "accept", + "note": "row 'photo.2024' resolves to photo.2024.jpg via _has_extension (dotted stems are not extensions) \u2014 the CLI's cross-check must mirror, not reject" + }, + { + "name": "tabular-na-labels", + "category": "tabular_classification", + "csv": "data.csv", + "label_column": "label", + "cli_verdict": "reject", + "ingestor_verdict": "reject", + "note": "'NA' drops to NaN for schema-typed labels (coercion NA_SENTINELS) \u2192 1 distinct class in-cluster; the CLI mirrors the sentinel drop" + }, + { + "name": "tabular-nonutf8", + "category": "tabular_classification", + "csv": "data.csv", + "label_column": "label", + "cli_verdict": "reject", + "ingestor_verdict": "reject", + "note": "check_csv_encoding runs before ANY validator in-cluster; Latin-1 exports must fail locally, not post-upload" + }, + { + "name": "text-clf-ok", + "category": "text_classification", + "csv": "labels.csv", + "label_column": "label", + "extension": ".txt", + "cli_verdict": "accept", + "ingestor_verdict": "accept", + "note": "pins the text-family dispatch" + } + ] +} diff --git a/internal/push/testdata/parity/cases/imgc-bom-labels/images/a.jpg b/internal/push/testdata/parity/cases/imgc-bom-labels/images/a.jpg new file mode 100644 index 0000000..2c5a1e7 Binary files /dev/null and b/internal/push/testdata/parity/cases/imgc-bom-labels/images/a.jpg differ diff --git a/internal/push/testdata/parity/cases/imgc-bom-labels/images/b.jpg b/internal/push/testdata/parity/cases/imgc-bom-labels/images/b.jpg new file mode 100644 index 0000000..2c5a1e7 Binary files /dev/null and b/internal/push/testdata/parity/cases/imgc-bom-labels/images/b.jpg differ diff --git a/internal/push/testdata/parity/cases/imgc-bom-labels/labels.csv b/internal/push/testdata/parity/cases/imgc-bom-labels/labels.csv new file mode 100644 index 0000000..6b226c6 --- /dev/null +++ b/internal/push/testdata/parity/cases/imgc-bom-labels/labels.csv @@ -0,0 +1,3 @@ +image_id,label +a.jpg,cat +b.jpg,dog diff --git a/internal/push/testdata/parity/cases/imgc-corrupt/images/a.jpg b/internal/push/testdata/parity/cases/imgc-corrupt/images/a.jpg new file mode 100644 index 0000000..2c5a1e7 Binary files /dev/null and b/internal/push/testdata/parity/cases/imgc-corrupt/images/a.jpg differ diff --git a/internal/push/testdata/parity/cases/imgc-corrupt/images/c.jpg b/internal/push/testdata/parity/cases/imgc-corrupt/images/c.jpg new file mode 100644 index 0000000..c14f8d5 --- /dev/null +++ b/internal/push/testdata/parity/cases/imgc-corrupt/images/c.jpg @@ -0,0 +1 @@ +garbage-not-an-image \ No newline at end of file diff --git a/internal/push/testdata/parity/cases/imgc-corrupt/labels.csv b/internal/push/testdata/parity/cases/imgc-corrupt/labels.csv new file mode 100644 index 0000000..0f87dcf --- /dev/null +++ b/internal/push/testdata/parity/cases/imgc-corrupt/labels.csv @@ -0,0 +1,3 @@ +image_id,label +a.jpg,cat +c.jpg,dog diff --git a/internal/push/testdata/parity/cases/imgc-dotted-stem/images/b.jpg b/internal/push/testdata/parity/cases/imgc-dotted-stem/images/b.jpg new file mode 100644 index 0000000..c6b07d9 Binary files /dev/null and b/internal/push/testdata/parity/cases/imgc-dotted-stem/images/b.jpg differ diff --git a/internal/push/testdata/parity/cases/imgc-dotted-stem/images/photo.2024.jpg b/internal/push/testdata/parity/cases/imgc-dotted-stem/images/photo.2024.jpg new file mode 100644 index 0000000..c6b07d9 Binary files /dev/null and b/internal/push/testdata/parity/cases/imgc-dotted-stem/images/photo.2024.jpg differ diff --git a/internal/push/testdata/parity/cases/imgc-dotted-stem/labels.csv b/internal/push/testdata/parity/cases/imgc-dotted-stem/labels.csv new file mode 100644 index 0000000..aefc6e8 --- /dev/null +++ b/internal/push/testdata/parity/cases/imgc-dotted-stem/labels.csv @@ -0,0 +1,3 @@ +image_id,label +photo.2024,cat +b.jpg,dog diff --git a/internal/push/testdata/parity/cases/imgc-dup-header/images/a.jpg b/internal/push/testdata/parity/cases/imgc-dup-header/images/a.jpg new file mode 100644 index 0000000..c6b07d9 Binary files /dev/null and b/internal/push/testdata/parity/cases/imgc-dup-header/images/a.jpg differ diff --git a/internal/push/testdata/parity/cases/imgc-dup-header/images/b.jpg b/internal/push/testdata/parity/cases/imgc-dup-header/images/b.jpg new file mode 100644 index 0000000..c6b07d9 Binary files /dev/null and b/internal/push/testdata/parity/cases/imgc-dup-header/images/b.jpg differ diff --git a/internal/push/testdata/parity/cases/imgc-dup-header/labels.csv b/internal/push/testdata/parity/cases/imgc-dup-header/labels.csv new file mode 100644 index 0000000..a55a8f7 --- /dev/null +++ b/internal/push/testdata/parity/cases/imgc-dup-header/labels.csv @@ -0,0 +1,3 @@ +image_id,label,label +a.jpg,cat,x +b.jpg,dog,y diff --git a/internal/push/testdata/parity/cases/imgc-empty-label/images/a.jpg b/internal/push/testdata/parity/cases/imgc-empty-label/images/a.jpg new file mode 100644 index 0000000..c6b07d9 Binary files /dev/null and b/internal/push/testdata/parity/cases/imgc-empty-label/images/a.jpg differ diff --git a/internal/push/testdata/parity/cases/imgc-empty-label/images/b.jpg b/internal/push/testdata/parity/cases/imgc-empty-label/images/b.jpg new file mode 100644 index 0000000..c6b07d9 Binary files /dev/null and b/internal/push/testdata/parity/cases/imgc-empty-label/images/b.jpg differ diff --git a/internal/push/testdata/parity/cases/imgc-empty-label/labels.csv b/internal/push/testdata/parity/cases/imgc-empty-label/labels.csv new file mode 100644 index 0000000..d6de19c --- /dev/null +++ b/internal/push/testdata/parity/cases/imgc-empty-label/labels.csv @@ -0,0 +1,3 @@ +image_id,label +a.jpg,A +b.jpg, diff --git a/internal/push/testdata/parity/cases/imgc-header-only/images/a.jpg b/internal/push/testdata/parity/cases/imgc-header-only/images/a.jpg new file mode 100644 index 0000000..2c5a1e7 Binary files /dev/null and b/internal/push/testdata/parity/cases/imgc-header-only/images/a.jpg differ diff --git a/internal/push/testdata/parity/cases/imgc-header-only/labels.csv b/internal/push/testdata/parity/cases/imgc-header-only/labels.csv new file mode 100644 index 0000000..6c4b18c --- /dev/null +++ b/internal/push/testdata/parity/cases/imgc-header-only/labels.csv @@ -0,0 +1 @@ +image_id,label diff --git a/internal/push/testdata/parity/cases/imgc-label-case/images/a.jpg b/internal/push/testdata/parity/cases/imgc-label-case/images/a.jpg new file mode 100644 index 0000000..2c5a1e7 Binary files /dev/null and b/internal/push/testdata/parity/cases/imgc-label-case/images/a.jpg differ diff --git a/internal/push/testdata/parity/cases/imgc-label-case/images/b.jpg b/internal/push/testdata/parity/cases/imgc-label-case/images/b.jpg new file mode 100644 index 0000000..2c5a1e7 Binary files /dev/null and b/internal/push/testdata/parity/cases/imgc-label-case/images/b.jpg differ diff --git a/internal/push/testdata/parity/cases/imgc-label-case/labels.csv b/internal/push/testdata/parity/cases/imgc-label-case/labels.csv new file mode 100644 index 0000000..0bf13b2 --- /dev/null +++ b/internal/push/testdata/parity/cases/imgc-label-case/labels.csv @@ -0,0 +1,3 @@ +image_id, Label +a.jpg,cat +b.jpg,dog diff --git a/internal/push/testdata/parity/cases/imgc-label-missing/images/a.jpg b/internal/push/testdata/parity/cases/imgc-label-missing/images/a.jpg new file mode 100644 index 0000000..2c5a1e7 Binary files /dev/null and b/internal/push/testdata/parity/cases/imgc-label-missing/images/a.jpg differ diff --git a/internal/push/testdata/parity/cases/imgc-label-missing/labels.csv b/internal/push/testdata/parity/cases/imgc-label-missing/labels.csv new file mode 100644 index 0000000..96a2f73 --- /dev/null +++ b/internal/push/testdata/parity/cases/imgc-label-missing/labels.csv @@ -0,0 +1,2 @@ +image_id,target +a.jpg,cat diff --git a/internal/push/testdata/parity/cases/imgc-label-uniform/images/a.jpg b/internal/push/testdata/parity/cases/imgc-label-uniform/images/a.jpg new file mode 100644 index 0000000..2c5a1e7 Binary files /dev/null and b/internal/push/testdata/parity/cases/imgc-label-uniform/images/a.jpg differ diff --git a/internal/push/testdata/parity/cases/imgc-label-uniform/images/b.jpg b/internal/push/testdata/parity/cases/imgc-label-uniform/images/b.jpg new file mode 100644 index 0000000..2c5a1e7 Binary files /dev/null and b/internal/push/testdata/parity/cases/imgc-label-uniform/images/b.jpg differ diff --git a/internal/push/testdata/parity/cases/imgc-label-uniform/labels.csv b/internal/push/testdata/parity/cases/imgc-label-uniform/labels.csv new file mode 100644 index 0000000..30cd31a --- /dev/null +++ b/internal/push/testdata/parity/cases/imgc-label-uniform/labels.csv @@ -0,0 +1,3 @@ +image_id,label +a.jpg,cat +b.jpg,cat diff --git a/internal/push/testdata/parity/cases/imgc-missing-file/images/a.jpg b/internal/push/testdata/parity/cases/imgc-missing-file/images/a.jpg new file mode 100644 index 0000000..2c5a1e7 Binary files /dev/null and b/internal/push/testdata/parity/cases/imgc-missing-file/images/a.jpg differ diff --git a/internal/push/testdata/parity/cases/imgc-missing-file/labels.csv b/internal/push/testdata/parity/cases/imgc-missing-file/labels.csv new file mode 100644 index 0000000..34fda8e --- /dev/null +++ b/internal/push/testdata/parity/cases/imgc-missing-file/labels.csv @@ -0,0 +1,3 @@ +image_id,label +a.jpg,cat +ghost.jpg,dog diff --git a/internal/push/testdata/parity/cases/imgc-nonsquare-swapped/images/a.jpg b/internal/push/testdata/parity/cases/imgc-nonsquare-swapped/images/a.jpg new file mode 100644 index 0000000..4b5275b Binary files /dev/null and b/internal/push/testdata/parity/cases/imgc-nonsquare-swapped/images/a.jpg differ diff --git a/internal/push/testdata/parity/cases/imgc-nonsquare-swapped/images/b.jpg b/internal/push/testdata/parity/cases/imgc-nonsquare-swapped/images/b.jpg new file mode 100644 index 0000000..4b5275b Binary files /dev/null and b/internal/push/testdata/parity/cases/imgc-nonsquare-swapped/images/b.jpg differ diff --git a/internal/push/testdata/parity/cases/imgc-nonsquare-swapped/labels.csv b/internal/push/testdata/parity/cases/imgc-nonsquare-swapped/labels.csv new file mode 100644 index 0000000..aa98ba5 --- /dev/null +++ b/internal/push/testdata/parity/cases/imgc-nonsquare-swapped/labels.csv @@ -0,0 +1,3 @@ +image_id,label +a.jpg,cat +b.jpg,dog diff --git a/internal/push/testdata/parity/cases/imgc-nonsquare/images/a.jpg b/internal/push/testdata/parity/cases/imgc-nonsquare/images/a.jpg new file mode 100644 index 0000000..4b5275b Binary files /dev/null and b/internal/push/testdata/parity/cases/imgc-nonsquare/images/a.jpg differ diff --git a/internal/push/testdata/parity/cases/imgc-nonsquare/images/b.jpg b/internal/push/testdata/parity/cases/imgc-nonsquare/images/b.jpg new file mode 100644 index 0000000..4b5275b Binary files /dev/null and b/internal/push/testdata/parity/cases/imgc-nonsquare/images/b.jpg differ diff --git a/internal/push/testdata/parity/cases/imgc-nonsquare/labels.csv b/internal/push/testdata/parity/cases/imgc-nonsquare/labels.csv new file mode 100644 index 0000000..aa98ba5 --- /dev/null +++ b/internal/push/testdata/parity/cases/imgc-nonsquare/labels.csv @@ -0,0 +1,3 @@ +image_id,label +a.jpg,cat +b.jpg,dog diff --git a/internal/push/testdata/parity/cases/imgc-ok/images/a.jpg b/internal/push/testdata/parity/cases/imgc-ok/images/a.jpg new file mode 100644 index 0000000..2c5a1e7 Binary files /dev/null and b/internal/push/testdata/parity/cases/imgc-ok/images/a.jpg differ diff --git a/internal/push/testdata/parity/cases/imgc-ok/images/b.jpg b/internal/push/testdata/parity/cases/imgc-ok/images/b.jpg new file mode 100644 index 0000000..2c5a1e7 Binary files /dev/null and b/internal/push/testdata/parity/cases/imgc-ok/images/b.jpg differ diff --git a/internal/push/testdata/parity/cases/imgc-ok/labels.csv b/internal/push/testdata/parity/cases/imgc-ok/labels.csv new file mode 100644 index 0000000..aa98ba5 --- /dev/null +++ b/internal/push/testdata/parity/cases/imgc-ok/labels.csv @@ -0,0 +1,3 @@ +image_id,label +a.jpg,cat +b.jpg,dog diff --git a/internal/push/testdata/parity/cases/imgc-res-mismatch/images/a.jpg b/internal/push/testdata/parity/cases/imgc-res-mismatch/images/a.jpg new file mode 100644 index 0000000..2c5a1e7 Binary files /dev/null and b/internal/push/testdata/parity/cases/imgc-res-mismatch/images/a.jpg differ diff --git a/internal/push/testdata/parity/cases/imgc-res-mismatch/images/odd.jpg b/internal/push/testdata/parity/cases/imgc-res-mismatch/images/odd.jpg new file mode 100644 index 0000000..3a59e2e Binary files /dev/null and b/internal/push/testdata/parity/cases/imgc-res-mismatch/images/odd.jpg differ diff --git a/internal/push/testdata/parity/cases/imgc-res-mismatch/labels.csv b/internal/push/testdata/parity/cases/imgc-res-mismatch/labels.csv new file mode 100644 index 0000000..08c8284 --- /dev/null +++ b/internal/push/testdata/parity/cases/imgc-res-mismatch/labels.csv @@ -0,0 +1,3 @@ +image_id,label +a.jpg,cat +odd.jpg,dog diff --git a/internal/push/testdata/parity/cases/imgc-zero-byte/images/a.jpg b/internal/push/testdata/parity/cases/imgc-zero-byte/images/a.jpg new file mode 100644 index 0000000..2c5a1e7 Binary files /dev/null and b/internal/push/testdata/parity/cases/imgc-zero-byte/images/a.jpg differ diff --git a/internal/push/testdata/parity/cases/imgc-zero-byte/images/z.jpg b/internal/push/testdata/parity/cases/imgc-zero-byte/images/z.jpg new file mode 100644 index 0000000..e69de29 diff --git a/internal/push/testdata/parity/cases/imgc-zero-byte/labels.csv b/internal/push/testdata/parity/cases/imgc-zero-byte/labels.csv new file mode 100644 index 0000000..d37fca0 --- /dev/null +++ b/internal/push/testdata/parity/cases/imgc-zero-byte/labels.csv @@ -0,0 +1,3 @@ +image_id,label +a.jpg,cat +z.jpg,dog diff --git a/internal/push/testdata/parity/cases/tabular-bom/data.csv b/internal/push/testdata/parity/cases/tabular-bom/data.csv new file mode 100644 index 0000000..5c76b18 --- /dev/null +++ b/internal/push/testdata/parity/cases/tabular-bom/data.csv @@ -0,0 +1,3 @@ +age,income,label +30,100,1 +40,200,0 diff --git a/internal/push/testdata/parity/cases/tabular-dup-header/data.csv b/internal/push/testdata/parity/cases/tabular-dup-header/data.csv new file mode 100644 index 0000000..b92f2f9 --- /dev/null +++ b/internal/push/testdata/parity/cases/tabular-dup-header/data.csv @@ -0,0 +1,3 @@ +age,income,age,label +1,2,3,x +4,5,6,y diff --git a/internal/push/testdata/parity/cases/tabular-header-only/data.csv b/internal/push/testdata/parity/cases/tabular-header-only/data.csv new file mode 100644 index 0000000..70ec026 --- /dev/null +++ b/internal/push/testdata/parity/cases/tabular-header-only/data.csv @@ -0,0 +1 @@ +age,income,label diff --git a/internal/push/testdata/parity/cases/tabular-label-missing/data.csv b/internal/push/testdata/parity/cases/tabular-label-missing/data.csv new file mode 100644 index 0000000..4f59854 --- /dev/null +++ b/internal/push/testdata/parity/cases/tabular-label-missing/data.csv @@ -0,0 +1,2 @@ +age,income +30,100 diff --git a/internal/push/testdata/parity/cases/tabular-na-labels/data.csv b/internal/push/testdata/parity/cases/tabular-na-labels/data.csv new file mode 100644 index 0000000..01fe6f8 --- /dev/null +++ b/internal/push/testdata/parity/cases/tabular-na-labels/data.csv @@ -0,0 +1,3 @@ +age,label +30,NA +40,x diff --git a/internal/push/testdata/parity/cases/tabular-nonutf8/data.csv b/internal/push/testdata/parity/cases/tabular-nonutf8/data.csv new file mode 100644 index 0000000..31e4f05 --- /dev/null +++ b/internal/push/testdata/parity/cases/tabular-nonutf8/data.csv @@ -0,0 +1,3 @@ +age,city,label +30,München,1 +40,Berlin,0 diff --git a/internal/push/testdata/parity/cases/tabular-ok/data.csv b/internal/push/testdata/parity/cases/tabular-ok/data.csv new file mode 100644 index 0000000..e7096ec --- /dev/null +++ b/internal/push/testdata/parity/cases/tabular-ok/data.csv @@ -0,0 +1,3 @@ +age,income,label +30,100,1 +40,200,0 diff --git a/internal/push/testdata/parity/cases/text-clf-ok/labels.csv b/internal/push/testdata/parity/cases/text-clf-ok/labels.csv new file mode 100644 index 0000000..ea6020a --- /dev/null +++ b/internal/push/testdata/parity/cases/text-clf-ok/labels.csv @@ -0,0 +1,3 @@ +filename,label +a.txt,pos +b.txt,neg diff --git a/internal/push/testdata/parity/cases/text-clf-ok/texts/a.txt b/internal/push/testdata/parity/cases/text-clf-ok/texts/a.txt new file mode 100644 index 0000000..0cf5ead --- /dev/null +++ b/internal/push/testdata/parity/cases/text-clf-ok/texts/a.txt @@ -0,0 +1 @@ +great product, would buy again \ No newline at end of file diff --git a/internal/push/testdata/parity/cases/text-clf-ok/texts/b.txt b/internal/push/testdata/parity/cases/text-clf-ok/texts/b.txt new file mode 100644 index 0000000..7244ad7 --- /dev/null +++ b/internal/push/testdata/parity/cases/text-clf-ok/texts/b.txt @@ -0,0 +1 @@ +terrible, do not want \ No newline at end of file diff --git a/internal/push/testdata/parity/goldens.json b/internal/push/testdata/parity/goldens.json new file mode 100644 index 0000000..033a73b --- /dev/null +++ b/internal/push/testdata/parity/goldens.json @@ -0,0 +1,127 @@ +{ + "_comment": "GENERATED by scripts/gen-validator-goldens.py from the REAL data-ingestors validators \u2014 do not hand-edit. Regenerate when the ingestor's rules change; parity_golden_test.go pins these.", + "verdicts": { + "imgc-bom-labels": { + "errors": [], + "verdict": "accept" + }, + "imgc-corrupt": { + "errors": [ + "ImageResolutionValidator: Files that could not be processed: ['images/c.jpg: not a valid image (corrupt or unsupported format)']" + ], + "verdict": "reject" + }, + "imgc-dotted-stem": { + "errors": [], + "verdict": "accept" + }, + "imgc-dup-header": { + "errors": [], + "verdict": "accept" + }, + "imgc-empty-label": { + "errors": [], + "verdict": "accept" + }, + "imgc-header-only": { + "errors": [ + "IngestableRecordsValidator: No data rows found in CSV 'labels.csv': the file has a header but no data rows (0 ingestable records). Add at least one data row and re-ingest." + ], + "verdict": "reject" + }, + "imgc-label-case": { + "errors": [], + "verdict": "accept" + }, + "imgc-label-missing": { + "errors": [ + "LabelColumnValidator: Configured label column 'label' not found in CSV header (columns: image_id, target). Set 'label:' in the ingest config to an existing column, or add the 'label' column to the CSV." + ], + "verdict": "reject" + }, + "imgc-label-uniform": { + "errors": [ + "LabelDiversityValidator: Classification category requires at least 2 distinct label values in column 'label' (after whitespace stripping); this dataset has 1 distinct value(s): ['cat']. Raw value counts: {'cat': 2}. If this is intentional (e.g. you have a continuous target), pick a regression-family category like tabular_regression or time_series_forecasting instead." + ], + "verdict": "reject" + }, + "imgc-missing-file": { + "errors": [], + "verdict": "accept" + }, + "imgc-nonsquare": { + "errors": [], + "verdict": "accept" + }, + "imgc-nonsquare-swapped": { + "errors": [ + "ImageResolutionValidator: Images with incorrect resolution found: ['images/b.jpg: (8, 4) (expected: [4, 8])', 'images/a.jpg: (8, 4) (expected: [4, 8])']" + ], + "verdict": "reject" + }, + "imgc-ok": { + "errors": [], + "verdict": "accept" + }, + "imgc-res-mismatch": { + "errors": [ + "ImageResolutionValidator: Multiple image resolutions found: [(4, 4), (8, 8)]. All images must have the same resolution.", + "ImageResolutionValidator: Expected resolution: [8, 8]", + "ImageResolutionValidator: Invalid files: []", + "ImageResolutionValidator: Resolution errors: ['images/odd.jpg: (4, 4) (expected: [8, 8])']" + ], + "verdict": "reject" + }, + "imgc-zero-byte": { + "errors": [ + "ImageResolutionValidator: Files that could not be processed: ['images/z.jpg: empty file (0 bytes)']" + ], + "verdict": "reject" + }, + "tabular-bom": { + "errors": [ + "DataValidator: Schema columns not present in CSV: age." + ], + "verdict": "reject" + }, + "tabular-dup-header": { + "errors": [ + "DataValidator: Duplicate column name(s) in the CSV header: ['age']. Each column must be unique \u2014 otherwise the second is silently renamed '.1' by the parser and the schema maps onto the wrong column." + ], + "verdict": "reject" + }, + "tabular-header-only": { + "errors": [ + "IngestableRecordsValidator: No data rows found in CSV 'data.csv': the file has a header but no data rows (0 ingestable records). Add at least one data row and re-ingest.", + "DataValidator: No data found to validate" + ], + "verdict": "reject" + }, + "tabular-label-missing": { + "errors": [], + "verdict": "accept" + }, + "tabular-na-labels": { + "errors": [ + "LabelDiversityValidator: Classification category requires at least 2 distinct label values in column 'label' (after whitespace stripping); this dataset has 1 distinct value(s): ['x']. Raw value counts: {'x': 1}. If this is intentional (e.g. you have a continuous target), pick a regression-family category like tabular_regression or time_series_forecasting instead." + ], + "verdict": "reject" + }, + "tabular-nonutf8": { + "errors": [ + "csv-encoding: \u001b[91m'data.csv' is not valid UTF-8 \u2014 a non-UTF-8 byte was found at byte 19. Re-save the file as UTF-8 (in Excel: Save As \u2192 'CSV UTF-8 (Comma delimited)'), then re-ingest.\u001b[0m", + "IngestableRecordsValidator: Ingestable records validation error: 'utf-8' codec can't decode byte 0xfc in position 1: invalid start byte", + "DataValidator: Data type validation error: 'utf-8' codec can't decode byte 0xfc in position 1: invalid start byte" + ], + "verdict": "reject" + }, + "tabular-ok": { + "errors": [], + "verdict": "accept" + }, + "text-clf-ok": { + "errors": [], + "verdict": "accept" + } + } +} diff --git a/scripts/gen-validator-goldens.py b/scripts/gen-validator-goldens.py new file mode 100644 index 0000000..5634e51 --- /dev/null +++ b/scripts/gen-validator-goldens.py @@ -0,0 +1,134 @@ +#!/usr/bin/env python3 +"""Regenerate internal/push/testdata/parity/goldens.json by running the REAL +data-ingestors validators over the parity fixture cases. + +This is the Python half of the validator-parity harness (backend#828 P3): +the CLI's local preflight (internal/push/preflight.go) previews the +in-cluster validators, and parity_golden_test.go pins that the two sides +agree case-by-case. When the ingestor's rules change, re-run this and +commit the diff — a changed verdict then fails the Go test until the +preview is consciously updated. + +Usage: + DATA_INGESTORS_DIR=~/path/to/data-ingestors python3 scripts/gen-validator-goldens.py + +DATA_INGESTORS_DIR must point at a checkout of tracebloc/data-ingestors +with its dependencies importable (its own .venv works: + ~/repos/data-ingestors/.venv/bin/python scripts/gen-validator-goldens.py +also does the trick — the script only needs pandas + Pillow, no MySQL). + +Verdicts are recorded at the accept/reject level, not message-text level — +error copy may drift harmlessly; verdicts may not. TableNameValidator and +DuplicateValidator are skipped (they check cluster-side state — the table +name is validated separately by both sides, and destination-duplicate +handling is the cli#70 guard's territory, not a data-hygiene rule). +""" + +import json +import os +import re +import sys + +HERE = os.path.dirname(os.path.abspath(__file__)) +PARITY = os.path.join(HERE, "..", "internal", "push", "testdata", "parity") + +di = os.environ.get("DATA_INGESTORS_DIR", "") +if di: + sys.path.insert(0, os.path.abspath(di)) + +try: + from tracebloc_ingestor.utils.validators_mapping import map_validators + from tracebloc_ingestor.config import Config + from tracebloc_ingestor.ingestors import preflight +except ImportError as exc: # pragma: no cover + sys.exit( + f"can't import the data-ingestors package ({exc}).\n" + "Set DATA_INGESTORS_DIR to a checkout, or run with its venv python." + ) + +SKIP = {"TableNameValidator", "DuplicateValidator"} +ANSI = re.compile(r"\x1b\[[0-9;]*m") + + +def infer_schema(csv_path): + """Mirror the CLI's inference shape: every trimmed header column typed — + the schema content doesn't matter for these validators, membership does. + Read with pandas (BOM-stripping) exactly like the CLI's header reader.""" + import pandas as pd + + cols = list(pd.read_csv(csv_path, nrows=0, encoding="utf-8").columns) + return {str(c).strip(): "VARCHAR(255)" for c in cols} + + +def run_case(case): + case_dir = os.path.join(PARITY, "cases", case["name"]) + csv_path = os.path.join(case_dir, case["csv"]) + + if True: # (kept indented for a small diff; no tempdir needed — DuplicateValidator is skipped) + cfg = Config(SRC_PATH=case_dir, TABLE_NAME="parity_t") + options = {"label_column": case.get("label_column", "label")} + if case.get("extension"): + options["extension"] = case["extension"] + if case.get("target_size"): + options["target_size"] = case["target_size"] + if case["category"].startswith(("tabular", "time_")): + try: + schema = infer_schema(csv_path) + except Exception: + schema = {} + options["schema"] = schema + options["full_schema"] = schema + + errors = [] + try: + preflight.check_csv_encoding(csv_path) + except Exception as exc: + errors.append(f"csv-encoding: {exc}") + + for v in map_validators(case["category"], options, cfg): + if type(v).__name__ in SKIP: + continue + try: + res = v.validate(csv_path) + if not res.is_valid: + errors.extend( + f"{type(v).__name__}: {ANSI.sub('', str(e)).replace(case_dir + os.sep, '')}" + for e in res.errors + ) + except Exception as exc: # a raising validator is a rejection too + errors.append(f"{type(v).__name__}: raised {exc}") + + return { + "verdict": "reject" if errors else "accept", + "errors": errors[:6], + } + + +def main(): + with open(os.path.join(PARITY, "cases.json")) as f: + manifest = json.load(f) + + goldens = {} + for case in manifest["cases"]: + goldens[case["name"]] = run_case(case) + print(f" {case['name']:24s} → {goldens[case['name']]['verdict']}") + + out = os.path.join(PARITY, "goldens.json") + with open(out, "w") as f: + json.dump( + { + "_comment": "GENERATED by scripts/gen-validator-goldens.py from the REAL " + "data-ingestors validators — do not hand-edit. Regenerate when the " + "ingestor's rules change; parity_golden_test.go pins these.", + "verdicts": goldens, + }, + f, + indent=2, + sort_keys=True, + ) + f.write("\n") + print(f"wrote {out}") + + +if __name__ == "__main__": + main() diff --git a/scripts/sync-validator-goldens.sh b/scripts/sync-validator-goldens.sh new file mode 100755 index 0000000..8cfc02a --- /dev/null +++ b/scripts/sync-validator-goldens.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# Regenerate (or --check) the validator-parity goldens against the REAL +# data-ingestors validators. Mirrors sync-schema.sh's shape. +# +# DATA_INGESTORS_DIR=~/repos/data-ingestors scripts/sync-validator-goldens.sh [--check] +# +# Needs a data-ingestors checkout with pandas + Pillow importable — its own +# .venv works: point PYTHON at it, e.g. +# PYTHON="$DATA_INGESTORS_DIR/.venv/bin/python" scripts/sync-validator-goldens.sh --check +set -euo pipefail +cd "$(dirname "$0")/.." +GOLDENS="internal/push/testdata/parity/goldens.json" +PYTHON="${PYTHON:-python3}" +if [[ "${1:-}" == "--check" ]]; then + tmp="$(mktemp -d)" + trap 'rm -rf "$tmp"' EXIT + cp "$GOLDENS" "$tmp/committed.json" + "$PYTHON" scripts/gen-validator-goldens.py >/dev/null + # Compare VERDICTS only — error text may drift harmlessly (and embeds + # fixture paths); verdicts may not. + if ! "$PYTHON" -c " +import json,sys +a=json.load(open('$tmp/committed.json'))['verdicts'] +b=json.load(open('$GOLDENS'))['verdicts'] +va={k:v['verdict'] for k,v in a.items()}; vb={k:v['verdict'] for k,v in b.items()} +sys.exit(0 if va==vb else 1) +"; then + cp "$tmp/committed.json" "$GOLDENS" # restore — check must not mutate + echo "DRIFT: the ingestor's validator verdicts changed. Re-run the generator," >&2 + echo "commit the new goldens, and update cases.json (+ the Go preview) consciously." >&2 + exit 1 + fi + cp "$tmp/committed.json" "$GOLDENS" # keep the committed copy (paths etc. unchanged) + echo "validator goldens in sync" +else + "$PYTHON" scripts/gen-validator-goldens.py +fi