examples/notebooks/multi_manager_example.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "USE_H5PY = False  # set to True to use h5py/hdf5lib instead\n",
    "if USE_H5PY:\n",
    "    import h5py\n",
    "    from h5py import MultiManager\n",
    "else:\n",
    "    import h5pyd as h5py  # Use the \"as\" syntax for code  compatibility\n",
    "    from h5pyd import MultiManager\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create a new file\n",
    "f = h5py.File(\"/home/test_user1/multi_try.h5\", mode=\"w\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create some datasets\n",
    "DSET_SHAPE = (10,)\n",
    "DSET_DTYPE = np.int32\n",
    "\n",
    "# create 4 datasets\n",
    "DSET_COUNT = 4\n",
    "datasets = []\n",
    "for i in range(DSET_COUNT):\n",
    "    dset = f.create_dataset(f\"dset_{i}\", shape=DSET_SHAPE, dtype=DSET_DTYPE)\n",
    "    datasets.append(dset)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data_in[0]: [0 1 2 3 4 5 6 7 8 9]\n",
      "data_in[1]: [100 101 102 103 104 105 106 107 108 109]\n",
      "data_in[2]: [200 201 202 203 204 205 206 207 208 209]\n",
      "data_in[3]: [300 301 302 303 304 305 306 307 308 309]\n"
     ]
    }
   ],
   "source": [
    "# initialize some data to write\n",
    "data_in = []\n",
    "for n in range(DSET_COUNT):\n",
    "    arr = np.zeros(DSET_SHAPE, dtype=DSET_DTYPE)\n",
    "    arr[...] = list(range(n*100, n*100+DSET_SHAPE[0]))\n",
    "    data_in.append(arr)\n",
    "    print(f\"data_in[{n}]: {arr}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# instantiate a MultiManager and use it to write to all the datasets simultaneously\n",
    "mm = MultiManager(datasets)\n",
    "mm[...] = data_in"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dset_0: [0 1 2 3 4 5 6 7 8 9]\n",
      "dset_1: [100 101 102 103 104 105 106 107 108 109]\n",
      "dset_2: [200 201 202 203 204 205 206 207 208 209]\n",
      "dset_3: [300 301 302 303 304 305 306 307 308 309]\n"
     ]
    }
   ],
   "source": [
    "# verify what get saved to each dataset\n",
    "for n in range(DSET_COUNT):\n",
    "    dset_name = f\"dset_{n}\"\n",
    "    dset = f[dset_name]\n",
    "    print(f\"{dset_name}: {dset[...]}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "4"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Read all the data from all the daasets using the same MultiManager instance\n",
    "data_out = mm[...]\n",
    "len(data_out)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data_out[0]: [0 1 2 3 4 5 6 7 8 9]\n",
      "data_out[1]: [100 101 102 103 104 105 106 107 108 109]\n",
      "data_out[2]: [200 201 202 203 204 205 206 207 208 209]\n",
      "data_out[3]: [300 301 302 303 304 305 306 307 308 309]\n"
     ]
    }
   ],
   "source": [
    "# dump the data that was returned\n",
    "for n in range(DSET_COUNT):\n",
    "    print(f\"data_out[{n}]: {data_out[n]}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data_out[0]: [0 1 2 3]\n",
      "data_out[1]: [100 101 102 103]\n",
      "data_out[2]: [200 201 202 203]\n",
      "data_out[3]: [300 301 302 303]\n"
     ]
    }
   ],
   "source": [
    "# rather than reading all the data for a dataset, you can read a given selection\n",
    "data_out = mm[0:4]\n",
    "for n in range(DSET_COUNT):\n",
    "    print(f\"data_out[{n}]: {data_out[n]}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# it's also possible to pass a list of selections and have each selection\n",
    "# read from the corresponding dataset\n",
    "selections = []\n",
    "for n in range(DSET_COUNT):\n",
    "    s = slice(n, n+2, 1)\n",
    "    selections.append(s)\n",
    "\n",
    "data_out = mm.__getitem__(selections)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "data_out[0]: [0 1]\n",
      "data_out[1]: [101 102]\n",
      "data_out[2]: [202 203]\n",
      "data_out[3]: [303 304]\n"
     ]
    }
   ],
   "source": [
    "# dump the data that was returned\n",
    "for n in range(DSET_COUNT):\n",
    "    print(f\"data_out[{n}]: {data_out[n]}\")"

   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}