support adcmod sibling directories

README.md

@@ -150,3 +150,9 @@ adc = parse_adc_file(bin_id, '/path/to/bin.adc', extended=True)
 ```
 
 Low-level stitching functions (`detect_pairs`, `stitch_pair`, `infill_stitched_image`) and raw extraction utilities (`extract_roi_images`, `extract_roi_image`) are available for specialized use cases.
+
+## Note: corrected ADC files (`adcmod`)
+
+Some datasets (e.g. MVCO) keep corrected ADC files outside the raw data directory so the raw data stays untouched. These live in an `adcmod` directory that is strictly a sibling of the raw data root, laid out as `adcmod/<day>/<pid>.adc.mod`, where `<day>` is the name of the directory containing the raw fileset. The `.adc.mod` format is byte-compatible with `.adc`.
+
+`ifcbkit` resolves these transparently: when listing or fetching a fileset it uses the corrected ADC in place of the raw `.adc` if one exists. Only the ADC file is substituted — `.hdr` and `.roi` always come from the raw data directory — and a raw `.adc` must still be present for the bin to be discovered.

src/ifcbkit/fileset.py

@@ -20,6 +20,45 @@
 DEFAULT_EXCLUDE = ['skip', 'beads']
 DEFAULT_INCLUDE = ['data']
 
+# Corrected ("modified") ADC files live in a directory named ``adcmod`` that is
+# a sibling of the raw data root directory, laid out as
+# ``adcmod/<day>/<pid>.adc.mod`` and byte-compatible with the raw ``.adc``.
+# Most datasets have no such sibling.
+ADCMOD_DIR = 'adcmod'
+ADCMOD_EXT = '.adc.mod'
+
+
+def _adcmod_path(fileset_dir, pid, root_path):
+    """Return the path a corrected ADC file would have for this fileset.
+
+    The ``adcmod`` directory is strictly a sibling of ``root_path``. The day
+    subdirectory name is the fileset's own containing directory name.
+
+    :param fileset_dir: directory containing the raw fileset
+    :param pid: the bin ID
+    :param root_path: the raw data root directory
+    """
+    day = os.path.basename(os.path.normpath(fileset_dir))
+    adcmod_root = os.path.join(
+        os.path.dirname(os.path.abspath(root_path)), ADCMOD_DIR)
+    return os.path.join(adcmod_root, day, pid + ADCMOD_EXT)
+
+
+def sync_resolve_adc_path(fileset_dir, pid, root_path):
+    """Return a corrected ``.adc.mod`` path if present, else the raw ``.adc``."""
+    cand = _adcmod_path(fileset_dir, pid, root_path)
+    if os.path.exists(cand):
+        return cand
+    return os.path.join(fileset_dir, pid + '.adc')
+
+
+async def async_resolve_adc_path(fileset_dir, pid, root_path):
+    """Return a corrected ``.adc.mod`` path if present, else the raw ``.adc``."""
+    cand = _adcmod_path(fileset_dir, pid, root_path)
+    if await aiopath.exists(cand):
+        return cand
+    return os.path.join(fileset_dir, pid + '.adc')
+
 
 def validate_path(
     filepath,
@@ -393,9 +432,12 @@ def paths(self, pid):
         exists, fs = self._exists(pid)
         if not exists:
             raise KeyError(pid)
+        adc = None
+        if self.require_adc:
+            adc = sync_resolve_adc_path(os.path.dirname(fs), os.path.basename(fs), self.root_path)
         return {
             'hdr': fs + '.hdr',
-            'adc': fs + '.adc' if self.require_adc else None,
+            'adc': adc,
             'roi': fs + '.roi' if self.require_roi else None,
         }
 
@@ -409,7 +451,7 @@ def list(self):
             yield {
                 'pid': bn,
                 'hdr': os.path.join(dp, bn + '.hdr'),
-                'adc': os.path.join(dp, bn + '.adc') if self.require_adc else None,
+                'adc': sync_resolve_adc_path(dp, bn, self.root_path) if self.require_adc else None,
                 'roi': os.path.join(dp, bn + '.roi') if self.require_roi else None,
             }
 
@@ -520,9 +562,12 @@ async def paths(self, pid):
         exists, fs = await self._exists(pid)
         if not exists:
             raise KeyError(pid)
+        adc = None
+        if self.require_adc:
+            adc = await async_resolve_adc_path(os.path.dirname(fs), os.path.basename(fs), self.root_path)
         return {
             'hdr': fs + '.hdr',
-            'adc': fs + '.adc' if self.require_adc else None,
+            'adc': adc,
             'roi': fs + '.roi' if self.require_roi else None,
         }
 
@@ -533,10 +578,13 @@ async def list(self):
             exclude=self.exclude, include=self.include,
             require_adc=self.require_adc, require_roi=self.require_roi,
         ):
+            adc = None
+            if self.require_adc:
+                adc = await async_resolve_adc_path(dp, bn, self.root_path)
             yield {
                 'pid': bn,
                 'hdr': os.path.join(dp, bn + '.hdr'),
-                'adc': os.path.join(dp, bn + '.adc') if self.require_adc else None,
+                'adc': adc,
                 'roi': os.path.join(dp, bn + '.roi') if self.require_roi else None,
             }
 

src/ifcbkit/stores/filesystem.py

@@ -2,13 +2,14 @@
 Filesystem-backed stores for IFCB bin data and ROI images.
 """
 
+import os
 from io import BytesIO
 
 import aiofiles
 
 from ..identifiers import parse_roi_id
 from ..fileset import (
-    async_find_fileset,
+    async_find_fileset, async_resolve_adc_path,
     SyncIfcbDataDirectory, AsyncIfcbDataDirectory,
     DEFAULT_INCLUDE, DEFAULT_EXCLUDE,
 )
@@ -41,6 +42,11 @@ async def _find_path(self, bin_id: str, ext: str) -> str | None:
         )
         if basepath is None:
             return None
+        if ext == 'adc':
+            return await async_resolve_adc_path(
+                os.path.dirname(basepath), os.path.basename(basepath),
+                self.root_path,
+            )
         return f"{basepath}.{ext}"
 
     async def exists(self, key: str) -> bool:

tests/test_fileset.py

@@ -0,0 +1,173 @@
+"""Tests for fileset path resolution, including corrected (adcmod) ADC files."""
+
+import asyncio
+import os
+import shutil
+
+import pytest
+
+from ifcbkit.fileset import SyncIfcbDataDirectory, AsyncIfcbDataDirectory
+
+PID = 'D20170426T164105_IFCB009'
+DAY = 'D20170426'
+
+# Real I-style fixture (the actual adcmod use case). Its containing directory
+# name is what the adcmod tree mirrors as the "day" subdirectory.
+I_PID = 'IFCB5_2012_028_081515'
+I_DAY = 'IFCB5_2012_028'
+I_FIXTURE_DIR = os.path.join(os.path.dirname(__file__), 'data', I_PID)
+
+
+def _copy_i_fileset(day_dir):
+    os.makedirs(day_dir, exist_ok=True)
+    for ext in ('hdr', 'adc', 'roi'):
+        shutil.copy(
+            os.path.join(I_FIXTURE_DIR, f'{I_PID}.{ext}'),
+            os.path.join(day_dir, f'{I_PID}.{ext}'),
+        )
+
+
+def _make_fileset(dirpath, pid):
+    os.makedirs(dirpath, exist_ok=True)
+    for ext in ('hdr', 'adc', 'roi'):
+        with open(os.path.join(dirpath, f'{pid}.{ext}'), 'w') as f:
+            f.write('')
+
+
+def _make_adcmod(adcmod_root, day, pid, content='mod'):
+    day_dir = os.path.join(adcmod_root, day)
+    os.makedirs(day_dir, exist_ok=True)
+    path = os.path.join(day_dir, f'{pid}.adc.mod')
+    with open(path, 'w') as f:
+        f.write(content)
+    return path
+
+
+def test_no_adcmod_uses_raw_adc(tmp_path):
+    root = tmp_path / 'data'
+    _make_fileset(str(root / DAY), PID)
+    dd = SyncIfcbDataDirectory(str(root))
+    assert dd.paths(PID)['adc'] == str(root / DAY / f'{PID}.adc')
+
+
+def test_adcmod_sibling_of_data_dir(tmp_path):
+    # <root>/data/<day>/<pid>.adc  and  <root>/adcmod/<day>/<pid>.adc.mod
+    root = tmp_path / 'data'
+    _make_fileset(str(root / DAY), PID)
+    mod = _make_adcmod(str(tmp_path / 'adcmod'), DAY, PID)
+    dd = SyncIfcbDataDirectory(str(root))
+    assert dd.paths(PID)['adc'] == mod
+
+
+def test_adcmod_with_year_level(tmp_path):
+    # <root>/data/2017/<day>/...  and  <root>/adcmod/<day>/<pid>.adc.mod
+    root = tmp_path / 'data'
+    _make_fileset(str(root / '2017' / DAY), PID)
+    mod = _make_adcmod(str(tmp_path / 'adcmod'), DAY, PID)
+    dd = SyncIfcbDataDirectory(str(root))
+    assert dd.paths(PID)['adc'] == mod
+
+
+def test_adcmod_appears_in_list(tmp_path):
+    root = tmp_path / 'data'
+    _make_fileset(str(root / DAY), PID)
+    mod = _make_adcmod(str(tmp_path / 'adcmod'), DAY, PID)
+    dd = SyncIfcbDataDirectory(str(root))
+    entries = list(dd.list())
+    assert len(entries) == 1
+    assert entries[0]['adc'] == mod
+
+
+def test_adcmod_inside_root_is_ignored(tmp_path):
+    # adcmod is strictly a sibling of root_path; one nested inside root (here a
+    # sibling of the intermediate year directory) must not be used.
+    root = tmp_path / 'data'
+    _make_fileset(str(root / '2017' / DAY), PID)
+    _make_adcmod(str(root / 'adcmod'), DAY, PID)
+    dd = SyncIfcbDataDirectory(str(root))
+    assert dd.paths(PID)['adc'] == str(root / '2017' / DAY / f'{PID}.adc')
+
+
+def test_adcmod_sibling_of_nonroot_ancestor_is_ignored(tmp_path):
+    # <root> is <tmp>/raw/data, so only <tmp>/raw/adcmod counts -- an adcmod
+    # sibling of <tmp>/raw's own parent must not be used.
+    root = tmp_path / 'raw' / 'data'
+    _make_fileset(str(root / DAY), PID)
+    _make_adcmod(str(tmp_path / 'adcmod'), DAY, PID)
+    dd = SyncIfcbDataDirectory(str(root))
+    assert dd.paths(PID)['adc'] == str(root / DAY / f'{PID}.adc')
+
+
+def test_adcmod_sibling_of_root_with_trailing_slash(tmp_path):
+    root = tmp_path / 'data'
+    _make_fileset(str(root / DAY), PID)
+    mod = _make_adcmod(str(tmp_path / 'adcmod'), DAY, PID)
+    dd = SyncIfcbDataDirectory(str(root) + os.sep)
+    assert dd.paths(PID)['adc'] == mod
+
+
+def test_async_adcmod_resolution(tmp_path):
+    root = tmp_path / 'data'
+    _make_fileset(str(root / DAY), PID)
+    mod = _make_adcmod(str(tmp_path / 'adcmod'), DAY, PID)
+    dd = AsyncIfcbDataDirectory(str(root))
+    paths = asyncio.run(dd.paths(PID))
+    assert paths['adc'] == mod
+
+
+# --- I-style bins (the real adcmod use case) ---
+
+def test_istyle_adcmod_path_resolves(tmp_path):
+    root = tmp_path / 'data'
+    _copy_i_fileset(str(root / I_DAY))
+    # corrected ADC is a byte-identical copy of the raw ADC
+    raw_adc = os.path.join(I_FIXTURE_DIR, f'{I_PID}.adc')
+    mod = _make_adcmod(
+        str(tmp_path / 'adcmod'), I_DAY, I_PID,
+        content=open(raw_adc).read(),
+    )
+    dd = SyncIfcbDataDirectory(str(root))
+    assert dd.paths(I_PID)['adc'] == mod
+
+
+def test_istyle_read_images_through_adcmod(tmp_path):
+    # End-to-end: I-style stitching must work reading the corrected ADC.
+    root = tmp_path / 'data'
+    _copy_i_fileset(str(root / I_DAY))
+    raw_adc = os.path.join(I_FIXTURE_DIR, f'{I_PID}.adc')
+
+    # baseline: raw ADC (no adcmod sibling)
+    baseline = SyncIfcbDataDirectory(str(root)).read_images(I_PID)
+    assert len(baseline) > 0
+
+    # with a byte-identical corrected ADC present, results must match
+    _make_adcmod(str(tmp_path / 'adcmod'), I_DAY, I_PID, content=open(raw_adc).read())
+    dd = SyncIfcbDataDirectory(str(root))
+    assert dd.paths(I_PID)['adc'].endswith('.adc.mod')
+    corrected = dd.read_images(I_PID)
+    assert set(corrected.keys()) == set(baseline.keys())
+    for t in baseline:
+        assert corrected[t].size == baseline[t].size
+
+
+def test_istyle_corrected_adc_content_is_used(tmp_path):
+    # Prove the corrected ADC (not the raw one) drives the output: zero out
+    # one ROI's width in the .adc.mod so that target drops from the results.
+    root = tmp_path / 'data'
+    _copy_i_fileset(str(root / I_DAY))
+    raw_adc = os.path.join(I_FIXTURE_DIR, f'{I_PID}.adc')
+
+    baseline = SyncIfcbDataDirectory(str(root)).read_images(I_PID)
+    dropped = sorted(baseline.keys())[0]
+
+    # I-style: width at column 11, height at 12 (0-based)
+    lines = open(raw_adc).read().splitlines()
+    fields = lines[dropped - 1].split(',')
+    fields[11] = '0'
+    fields[12] = '0'
+    lines[dropped - 1] = ','.join(fields)
+    _make_adcmod(str(tmp_path / 'adcmod'), I_DAY, I_PID, content='\n'.join(lines) + '\n')
+
+    corrected = SyncIfcbDataDirectory(str(root)).read_images(I_PID)
+    assert dropped in baseline
+    assert dropped not in corrected

tests/test_stores.py

@@ -0,0 +1,73 @@
+"""Tests for filesystem-backed stores, including corrected (adcmod) ADC files."""
+
+import asyncio
+import os
+
+from ifcbkit.stores.filesystem import AsyncFilesystemBinStore
+
+PID = 'D20170426T164105_IFCB009'
+DAY = 'D20170426'
+
+
+def _make_fileset(dirpath, pid):
+    os.makedirs(dirpath, exist_ok=True)
+    for ext in ('hdr', 'adc', 'roi'):
+        with open(os.path.join(dirpath, f'{pid}.{ext}'), 'w') as f:
+            f.write('')
+
+
+def _make_adcmod(adcmod_root, day, pid, content='mod'):
+    day_dir = os.path.join(adcmod_root, day)
+    os.makedirs(day_dir, exist_ok=True)
+    path = os.path.join(day_dir, f'{pid}.adc.mod')
+    with open(path, 'w') as f:
+        f.write(content)
+    return path
+
+
+def test_bin_store_no_adcmod_uses_raw_adc(tmp_path):
+    root = tmp_path / 'data'
+    _make_fileset(str(root / DAY), PID)
+    store = AsyncFilesystemBinStore(str(root))
+    path = asyncio.run(store.get_path(f'{PID}.adc'))
+    assert path == str(root / DAY / f'{PID}.adc')
+
+
+def test_bin_store_resolves_adcmod(tmp_path):
+    root = tmp_path / 'data'
+    _make_fileset(str(root / DAY), PID)
+    mod = _make_adcmod(str(tmp_path / 'adcmod'), DAY, PID)
+    store = AsyncFilesystemBinStore(str(root))
+    assert asyncio.run(store.get_path(f'{PID}.adc')) == mod
+
+
+def test_bin_store_get_reads_adcmod_content(tmp_path):
+    root = tmp_path / 'data'
+    _make_fileset(str(root / DAY), PID)
+    _make_adcmod(str(tmp_path / 'adcmod'), DAY, PID, content='corrected')
+    store = AsyncFilesystemBinStore(str(root))
+    assert asyncio.run(store.get(f'{PID}.adc')) == b'corrected'
+
+
+def test_bin_store_adcmod_does_not_affect_hdr_or_roi(tmp_path):
+    root = tmp_path / 'data'
+    _make_fileset(str(root / DAY), PID)
+    _make_adcmod(str(tmp_path / 'adcmod'), DAY, PID)
+    store = AsyncFilesystemBinStore(str(root))
+    assert asyncio.run(store.get_path(f'{PID}.hdr')) == str(root / DAY / f'{PID}.hdr')
+    assert asyncio.run(store.get_path(f'{PID}.roi')) == str(root / DAY / f'{PID}.roi')
+
+
+def test_bin_store_adcmod_with_year_level(tmp_path):
+    root = tmp_path / 'data'
+    _make_fileset(str(root / '2017' / DAY), PID)
+    mod = _make_adcmod(str(tmp_path / 'adcmod'), DAY, PID)
+    store = AsyncFilesystemBinStore(str(root))
+    assert asyncio.run(store.get_path(f'{PID}.adc')) == mod
+
+
+def test_bin_store_exists_unknown_bin(tmp_path):
+    root = tmp_path / 'data'
+    _make_fileset(str(root / DAY), PID)
+    store = AsyncFilesystemBinStore(str(root))
+    assert asyncio.run(store.get_path('D20991231T000000_IFCB009.adc')) is None