From a7c005aaff083bc6e80107656cac1e133cdbf08e Mon Sep 17 00:00:00 2001 From: claw Date: Wed, 20 May 2026 11:22:27 +0000 Subject: [PATCH 1/6] docs: add examples tying the various data loaders and explaining their hierarchy --- examples/README.md | 1 + examples/loaders_101/data_loader_hierarchy.py | 106 ++++++++++++++++++ 2 files changed, 107 insertions(+) create mode 100644 examples/loaders_101/data_loader_hierarchy.py diff --git a/examples/README.md b/examples/README.md index d1cb72d1..8dbb399f 100644 --- a/examples/README.md +++ b/examples/README.md @@ -16,6 +16,7 @@ The following files are under *loaders_101* | File | Description | instructions | | -----| ------------| -----| | loaders.ipynb | A notebook with some sample code for aperturedb | Also available to read at [Aperturedb documentation](https://docs.aperturedata.io/HowToGuides/Advanced/loaders)| +| data_loader_hierarchy.py | A script explaining the data loader hierarchy and tying them together | ``python data_loader_hierarchy.py``| ## Example 2: Image classification using a pretrained model The following files are under *image_classification* diff --git a/examples/loaders_101/data_loader_hierarchy.py b/examples/loaders_101/data_loader_hierarchy.py new file mode 100644 index 00000000..24faefab --- /dev/null +++ b/examples/loaders_101/data_loader_hierarchy.py @@ -0,0 +1,106 @@ +""" +Data Loader Hierarchy and Examples + +ApertureDB python SDK uses `ParallelLoader` as the main mechanism to efficiently batch and load data in parallel. +The `ParallelLoader` relies on the `Subscriptable` interface. +Classes that inherit from `Subscriptable` can be passed to `ParallelLoader.ingest()`. + +The main data loaders provided by ApertureDB handle parsing CSVs and generating the appropriate ApertureDB queries. +The hierarchy is as follows: + +Subscriptable + └── CSVParser + ├── EntityDataCSV (Loads entities and their properties) + ├── ImageDataCSV (Loads images as entities and uploads image blobs) + ├── VideoDataCSV (Loads videos) + ├── BlobDataCSV (Loads generic blobs) + ├── DescriptorDataCSV (Loads descriptors, e.g., embeddings) + ├── ConnectionDataCSV (Loads connections between entities) + ├── BBoxDataCSV (Loads bounding boxes) + └── PolygonDataCSV (Loads polygons) + +Other non-CSV data loaders (like PyTorchData, KaggleData) also inherit from `Subscriptable`. + +The following script demonstrates how to instantiate different loaders and tie them together using `ParallelLoader`. +""" + +import os +import pandas as pd +from aperturedb.Connector import Connector +from aperturedb.ParallelLoader import ParallelLoader +from aperturedb.EntityDataCSV import EntityDataCSV +from aperturedb.ImageDataCSV import ImageDataCSV +from aperturedb.ConnectionDataCSV import ConnectionDataCSV + +def create_sample_csvs(): + # 1. Create a sample CSV for Entities (Persons) + df_persons = pd.DataFrame({ + "name": ["Alice", "Bob"], + "age": [25, 30] + }) + df_persons.to_csv("persons.csv", index=False) + + # 2. Create a sample CSV for Images + # Note: the paths should point to real images in a real scenario + with open("dummy_image.jpg", "wb") as f: + f.write(b"dummy_image_data") + + df_images = pd.DataFrame({ + "url": ["dummy_image.jpg", "dummy_image.jpg"], + "source": ["camera1", "camera2"] + }) + df_images.to_csv("images.csv", index=False) + + # 3. Create a sample CSV for Connections (Person -> Image) + df_connections = pd.DataFrame({ + "src_name": ["Alice", "Bob"], + "dst_url": ["dummy_image.jpg", "dummy_image.jpg"], + "connection_property": ["owns", "likes"] + }) + df_connections.to_csv("connections.csv", index=False) + +def main(): + create_sample_csvs() + + # Connect to ApertureDB (Make sure your ApertureDB instance is running) + db = Connector() + + # Initialize the ParallelLoader + # The ParallelLoader handles the actual ingestion of queries produced by the Data Loader objects + loader = ParallelLoader(db) + + # 1. Load Entities + print("Loading Entities...") + # By providing the kwargs like `name`, `age`, we map the columns to properties. + person_loader = EntityDataCSV("persons.csv", entity_class="Person", name="name", age="age") + loader.ingest(person_loader) + + # 2. Load Images + print("Loading Images...") + image_loader = ImageDataCSV("images.csv", source="source") + loader.ingest(image_loader) + + # 3. Load Connections + print("Loading Connections...") + connection_loader = ConnectionDataCSV( + "connections.csv", + connection_class="HasImage", + src_class="Person", + dst_class="Image", + src_prop_key="name", # Match the 'name' column in persons to 'src_name' + src_prop_val="src_name", + dst_prop_key="url", # Match the 'url' column in images to 'dst_url' + dst_prop_val="dst_url", + connection_property="connection_property" + ) + loader.ingest(connection_loader) + print("Done loading data!") + + # Cleanup local files + os.remove("persons.csv") + os.remove("images.csv") + os.remove("connections.csv") + os.remove("dummy_image.jpg") + +if __name__ == "__main__": + main() From 0a2b1e5100b49832296572478163835ae433d508 Mon Sep 17 00:00:00 2001 From: claw Date: Wed, 20 May 2026 11:46:46 +0000 Subject: [PATCH 2/6] fix: pre-commit issues --- examples/loaders_101/data_loader_hierarchy.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/examples/loaders_101/data_loader_hierarchy.py b/examples/loaders_101/data_loader_hierarchy.py index 24faefab..7ce6464a 100644 --- a/examples/loaders_101/data_loader_hierarchy.py +++ b/examples/loaders_101/data_loader_hierarchy.py @@ -32,6 +32,7 @@ from aperturedb.ImageDataCSV import ImageDataCSV from aperturedb.ConnectionDataCSV import ConnectionDataCSV + def create_sample_csvs(): # 1. Create a sample CSV for Entities (Persons) df_persons = pd.DataFrame({ @@ -44,7 +45,7 @@ def create_sample_csvs(): # Note: the paths should point to real images in a real scenario with open("dummy_image.jpg", "wb") as f: f.write(b"dummy_image_data") - + df_images = pd.DataFrame({ "url": ["dummy_image.jpg", "dummy_image.jpg"], "source": ["camera1", "camera2"] @@ -59,11 +60,12 @@ def create_sample_csvs(): }) df_connections.to_csv("connections.csv", index=False) + def main(): create_sample_csvs() # Connect to ApertureDB (Make sure your ApertureDB instance is running) - db = Connector() + db = Connector() # Initialize the ParallelLoader # The ParallelLoader handles the actual ingestion of queries produced by the Data Loader objects @@ -72,7 +74,8 @@ def main(): # 1. Load Entities print("Loading Entities...") # By providing the kwargs like `name`, `age`, we map the columns to properties. - person_loader = EntityDataCSV("persons.csv", entity_class="Person", name="name", age="age") + person_loader = EntityDataCSV( + "persons.csv", entity_class="Person", name="name", age="age") loader.ingest(person_loader) # 2. Load Images @@ -102,5 +105,6 @@ def main(): os.remove("connections.csv") os.remove("dummy_image.jpg") + if __name__ == "__main__": main() From c67eb678c4f16db1a9b1b0a9edbe61c1c65584b1 Mon Sep 17 00:00:00 2001 From: claw Date: Wed, 20 May 2026 11:48:13 +0000 Subject: [PATCH 3/6] fix: remove trailing whitespace --- examples/loaders_101/data_loader_hierarchy.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/examples/loaders_101/data_loader_hierarchy.py b/examples/loaders_101/data_loader_hierarchy.py index 7ce6464a..35855f6e 100644 --- a/examples/loaders_101/data_loader_hierarchy.py +++ b/examples/loaders_101/data_loader_hierarchy.py @@ -1,8 +1,8 @@ """ Data Loader Hierarchy and Examples -ApertureDB python SDK uses `ParallelLoader` as the main mechanism to efficiently batch and load data in parallel. -The `ParallelLoader` relies on the `Subscriptable` interface. +ApertureDB python SDK uses `ParallelLoader` as the main mechanism to efficiently batch and load data in parallel. +The `ParallelLoader` relies on the `Subscriptable` interface. Classes that inherit from `Subscriptable` can be passed to `ParallelLoader.ingest()`. The main data loaders provided by ApertureDB handle parsing CSVs and generating the appropriate ApertureDB queries. From 3887f37e229c9e05130b8262633621d00cd3a6b2 Mon Sep 17 00:00:00 2001 From: claw Date: Wed, 20 May 2026 21:21:14 +0000 Subject: [PATCH 4/6] Fix example data loader hierarchy script based on review comments - Add EntityClass column for EntityDataCSV - Generate a real valid tiny PNG image instead of dummy bytes - Use filename instead of url for local images in ImageDataCSV - Fix ConnectionDataCSV columns to conform to expected @ format - Clean up temporary files safely using tempfile.TemporaryDirectory() --- examples/loaders_101/data_loader_hierarchy.py | 89 ++++++++++--------- 1 file changed, 45 insertions(+), 44 deletions(-) diff --git a/examples/loaders_101/data_loader_hierarchy.py b/examples/loaders_101/data_loader_hierarchy.py index 35855f6e..d5df1c6a 100644 --- a/examples/loaders_101/data_loader_hierarchy.py +++ b/examples/loaders_101/data_loader_hierarchy.py @@ -25,6 +25,8 @@ """ import os +import base64 +import tempfile import pandas as pd from aperturedb.Connector import Connector from aperturedb.ParallelLoader import ParallelLoader @@ -33,37 +35,46 @@ from aperturedb.ConnectionDataCSV import ConnectionDataCSV -def create_sample_csvs(): +def create_sample_csvs(base_dir): # 1. Create a sample CSV for Entities (Persons) df_persons = pd.DataFrame({ + "EntityClass": ["Person", "Person"], "name": ["Alice", "Bob"], "age": [25, 30] }) - df_persons.to_csv("persons.csv", index=False) + persons_csv = os.path.join(base_dir, "persons.csv") + df_persons.to_csv(persons_csv, index=False) # 2. Create a sample CSV for Images # Note: the paths should point to real images in a real scenario - with open("dummy_image.jpg", "wb") as f: - f.write(b"dummy_image_data") + dummy_image_path = os.path.join(base_dir, "dummy_image.png") + # 1x1 transparent PNG base64 + tiny_png_b64 = "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAQAAAC1HAwCAAAAC0lEQVR42mNkYAAAAAYAAjCB0C8AAAAASUVORK5CYII=" + with open(dummy_image_path, "wb") as f: + f.write(base64.b64decode(tiny_png_b64)) df_images = pd.DataFrame({ - "url": ["dummy_image.jpg", "dummy_image.jpg"], + "filename": [dummy_image_path, dummy_image_path], + "image_id": ["img1", "img2"], "source": ["camera1", "camera2"] }) - df_images.to_csv("images.csv", index=False) + images_csv = os.path.join(base_dir, "images.csv") + df_images.to_csv(images_csv, index=False) # 3. Create a sample CSV for Connections (Person -> Image) df_connections = pd.DataFrame({ - "src_name": ["Alice", "Bob"], - "dst_url": ["dummy_image.jpg", "dummy_image.jpg"], + "ConnectionClass": ["HasImage", "HasImage"], + "Person@name": ["Alice", "Bob"], + "_Image@image_id": ["img1", "img2"], "connection_property": ["owns", "likes"] }) - df_connections.to_csv("connections.csv", index=False) + connections_csv = os.path.join(base_dir, "connections.csv") + df_connections.to_csv(connections_csv, index=False) + + return persons_csv, images_csv, connections_csv def main(): - create_sample_csvs() - # Connect to ApertureDB (Make sure your ApertureDB instance is running) db = Connector() @@ -71,39 +82,29 @@ def main(): # The ParallelLoader handles the actual ingestion of queries produced by the Data Loader objects loader = ParallelLoader(db) - # 1. Load Entities - print("Loading Entities...") - # By providing the kwargs like `name`, `age`, we map the columns to properties. - person_loader = EntityDataCSV( - "persons.csv", entity_class="Person", name="name", age="age") - loader.ingest(person_loader) - - # 2. Load Images - print("Loading Images...") - image_loader = ImageDataCSV("images.csv", source="source") - loader.ingest(image_loader) - - # 3. Load Connections - print("Loading Connections...") - connection_loader = ConnectionDataCSV( - "connections.csv", - connection_class="HasImage", - src_class="Person", - dst_class="Image", - src_prop_key="name", # Match the 'name' column in persons to 'src_name' - src_prop_val="src_name", - dst_prop_key="url", # Match the 'url' column in images to 'dst_url' - dst_prop_val="dst_url", - connection_property="connection_property" - ) - loader.ingest(connection_loader) - print("Done loading data!") - - # Cleanup local files - os.remove("persons.csv") - os.remove("images.csv") - os.remove("connections.csv") - os.remove("dummy_image.jpg") + with tempfile.TemporaryDirectory() as temp_dir: + persons_csv, images_csv, connections_csv = create_sample_csvs(temp_dir) + + # 1. Load Entities + print("Loading Entities...") + # By providing the kwargs like `name`, `age`, we map the columns to properties. + person_loader = EntityDataCSV( + persons_csv, name="name", age="age") + loader.ingest(person_loader) + + # 2. Load Images + print("Loading Images...") + image_loader = ImageDataCSV(images_csv, image_id="image_id", source="source") + loader.ingest(image_loader) + + # 3. Load Connections + print("Loading Connections...") + connection_loader = ConnectionDataCSV( + connections_csv, + connection_property="connection_property" + ) + loader.ingest(connection_loader) + print("Done loading data!") if __name__ == "__main__": From 33642f6b482a6c1e2ac79c0193c20e613c3b153e Mon Sep 17 00:00:00 2001 From: claw Date: Thu, 21 May 2026 04:18:32 +0000 Subject: [PATCH 5/6] Fix pre-commit issues --- examples/loaders_101/data_loader_hierarchy.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/examples/loaders_101/data_loader_hierarchy.py b/examples/loaders_101/data_loader_hierarchy.py index d5df1c6a..f5e33842 100644 --- a/examples/loaders_101/data_loader_hierarchy.py +++ b/examples/loaders_101/data_loader_hierarchy.py @@ -70,7 +70,7 @@ def create_sample_csvs(base_dir): }) connections_csv = os.path.join(base_dir, "connections.csv") df_connections.to_csv(connections_csv, index=False) - + return persons_csv, images_csv, connections_csv @@ -94,7 +94,8 @@ def main(): # 2. Load Images print("Loading Images...") - image_loader = ImageDataCSV(images_csv, image_id="image_id", source="source") + image_loader = ImageDataCSV( + images_csv, image_id="image_id", source="source") loader.ingest(image_loader) # 3. Load Connections From 524985f90cfb9bf7d91fc086f66428eec6b7e0e1 Mon Sep 17 00:00:00 2001 From: ad-claw000 Date: Thu, 21 May 2026 10:28:50 +0000 Subject: [PATCH 6/6] Fix data_loader_hierarchy.py: add constraints to CSVs and remove unused kwargs --- examples/loaders_101/data_loader_hierarchy.py | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/examples/loaders_101/data_loader_hierarchy.py b/examples/loaders_101/data_loader_hierarchy.py index f5e33842..ef0635f0 100644 --- a/examples/loaders_101/data_loader_hierarchy.py +++ b/examples/loaders_101/data_loader_hierarchy.py @@ -40,7 +40,8 @@ def create_sample_csvs(base_dir): df_persons = pd.DataFrame({ "EntityClass": ["Person", "Person"], "name": ["Alice", "Bob"], - "age": [25, 30] + "age": [25, 30], + "constraint_name": ["Alice", "Bob"] }) persons_csv = os.path.join(base_dir, "persons.csv") df_persons.to_csv(persons_csv, index=False) @@ -56,7 +57,8 @@ def create_sample_csvs(base_dir): df_images = pd.DataFrame({ "filename": [dummy_image_path, dummy_image_path], "image_id": ["img1", "img2"], - "source": ["camera1", "camera2"] + "source": ["camera1", "camera2"], + "constraint_image_id": ["img1", "img2"] }) images_csv = os.path.join(base_dir, "images.csv") df_images.to_csv(images_csv, index=False) @@ -87,23 +89,18 @@ def main(): # 1. Load Entities print("Loading Entities...") - # By providing the kwargs like `name`, `age`, we map the columns to properties. - person_loader = EntityDataCSV( - persons_csv, name="name", age="age") + # Properties are derived directly from the CSV header names. + person_loader = EntityDataCSV(persons_csv) loader.ingest(person_loader) # 2. Load Images print("Loading Images...") - image_loader = ImageDataCSV( - images_csv, image_id="image_id", source="source") + image_loader = ImageDataCSV(images_csv) loader.ingest(image_loader) # 3. Load Connections print("Loading Connections...") - connection_loader = ConnectionDataCSV( - connections_csv, - connection_property="connection_property" - ) + connection_loader = ConnectionDataCSV(connections_csv) loader.ingest(connection_loader) print("Done loading data!")