Use merlin-dataloader package

.github/workflows/tensorflow.yml

@@ -43,6 +43,7 @@ jobs:
           fi
           pip install "pandas>=1.2.0,<1.4.0dev0"
           pip install "NVTabular@git+https://github.com/NVIDIA-Merlin/NVTabular.git@$branch"
+          pip install "merlin-dataloader@git+https://github.com/NVIDIA-Merlin/dataloader.git@$branch"
           pip install "merlin-core@git+https://github.com/NVIDIA-Merlin/core.git@$branch"
       - name: Install dependencies
         run: |
@@ -108,6 +109,7 @@ jobs:
           fi
           pip install "pandas>=1.2.0,<1.4.0dev0"
           pip install "NVTabular@git+https://github.com/NVIDIA-Merlin/NVTabular.git@$branch"
+          pip install "merlin-dataloader@git+https://github.com/NVIDIA-Merlin/dataloader.git@$branch"
           pip install "merlin-core@git+https://github.com/NVIDIA-Merlin/core.git@$branch"
       - name: Install dependencies
         run: |

examples/usecases/ecommerce-session-based-next-item-prediction-for-fashion.ipynb

@@ -968,8 +968,8 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "loader = mm.Loader(train, batch_size=BATCH_SIZE, transform=mm.ToTarget(train.schema, \"purchase_id_first\", one_hot=True),  shuffle = False)\n",
-    "val_loader = mm.Loader(valid, batch_size=BATCH_SIZE, transform=mm.ToTarget(train.schema, \"purchase_id_first\", one_hot=True),  shuffle=False)"
+    "loader = mm.Loader(train, batch_size=BATCH_SIZE, shuffle=False).map(mm.ToTarget(train.schema, \"purchase_id_first\", one_hot=True))\n",
+    "val_loader = mm.Loader(valid, batch_size=BATCH_SIZE, shuffle=False).map(mm.ToTarget(train.schema, \"purchase_id_first\", one_hot=True))"
    ]
   },
   {
@@ -1546,9 +1546,13 @@
     }
    ],
    "source": [
+    "def as_ragged(inputs, targets):\n",
+    "    _as_ragged = mm.ListToRagged()\n",
+    "    return _as_ragged(inputs), targets\n",
+    "\n",
     "history = model_bi_lstm.fit(\n",
-    "    loader,\n",
-    "    validation_data=val_loader,\n",
+    "    loader.map(as_ragged),\n",
+    "    validation_data=val_loader.map(as_ragged),\n",
     "    epochs=EPOCHS,\n",
     ")"
    ]

merlin/datasets/synthetic.py

@@ -25,7 +25,7 @@
 
 import merlin.io
 from merlin.models.utils import schema_utils
-from merlin.schema import Schema, Tags
+from merlin.schema import ColumnSchema, Schema, Tags
 from merlin.schema.io.tensorflow_metadata import TensorflowMetadata
 
 LOG = logging.getLogger("merlin-models")
@@ -116,6 +116,21 @@ def generate_data(
     else:
         raise ValueError(f"Unknown input type: {type(input)}")
 
+    for col in schema.column_names:
+        if not schema[col].is_list:
+            continue
+        new_properties = schema[col].properties
+        new_properties["value_count"] = {"min": min_session_length}
+        if max_session_length:
+            new_properties["value_count"]["max"] = max_session_length
+        schema[col] = ColumnSchema(
+            name=schema[col].name,
+            tags=schema[col].tags,
+            properties=new_properties,
+            dtype=schema[col].dtype,
+            is_list=True,
+        )
+
     df = generate_user_item_interactions(
         schema, num_rows, min_session_length, max_session_length, device=device
     )