Feature/convert and load

.github/workflows/python-cicd.yml

@@ -23,13 +23,9 @@ jobs:
         python -m pip install --upgrade pip
         if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
 
-    - name: Run Jupyter notebook tests
+    - name: Run all tests
       run: |
         export PYTHONPATH="$GITHUB_WORKSPACE"
-        make testnb
-
-    - name: Run Python tests
-      run: |
         make test
 
     - name: Lint with pylint

Makefile

@@ -1,12 +1,14 @@
-all: init test pylint
+all: init testnb test pylint
 
 init:
 	pip install -r requirements.txt
 
-test:
+test: test-nb test-py
+
+test-py:
 	nosetests --with-coverage -v --cover-package=tfrecorder
 
-testnb:
+test-nb:
 	ls -1 samples/*.ipynb | grep -v '^.*Dataflow.ipynb' | xargs py.test --nbval-lax -p no:python
 
 pylint:

README.md

@@ -109,7 +109,7 @@ Using Python interpreter:
 ```python
 import tfrecorder
 
-tfrecorder.create_tfrecords(
+tfrecorder.convert(
     source='/path/to/data.csv',
     output_dir='gs://my/bucket')
 ```
@@ -126,10 +126,9 @@ tfrecorder create-tfrecords \
 ```python
 import tfrecorder
 
-tfrecorder.create_tfrecords(
+tfrecorder.convert(
     source='/path/to/image_dir',
-    output_dir='gs://my/bucket',
-)
+    output_dir='gs://my/bucket')
 ```
 
 The image directory should have the following general structure:
@@ -175,8 +174,9 @@ Using Python interpreter:
 ```python
 import tfrecorder
 
-tfrecorder.check_tfrecords(
-    file_pattern='/path/to/tfrecords/train*.tfrecord.gz',
+tfrecorder.inspect(
+    tfrecord_dir='/path/to/tfrecords/',
+    split='TRAIN',
     num_records=5,
     output_dir='/tmp/output')
 ```
@@ -187,8 +187,9 @@ representing the images encoded into TFRecords.
 Using the command line:
 
 ```bash
-tfrecorder check-tfrecords \
-    --file_pattern=/path/to/tfrecords/train*.tfrecord.gz \
+tfrecorder inspect \
+    --tfrecord-dir=/path/to/tfrecords/ \
+    --split='TRAIN' \
     --num_records=5 \
     --output_dir=/tmp/output
 ```

samples/Convert-image-directory.ipynb

@@ -9,28 +9,27 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": 1,
    "metadata": {},
-   "outputs": [
-    {
-     "name": "stdout",
-     "output_type": "stream",
-     "text": [
-      "The autoreload extension is already loaded. To reload it, use:\n",
-      "  %reload_ext autoreload\n"
-     ]
-    }
-   ],
+   "outputs": [],
    "source": [
     "%load_ext autoreload\n",
     "%autoreload 2"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": 2,
    "metadata": {},
-   "outputs": [],
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Error importing tfx_bsl_extension.arrow.array_util. Some tfx_bsl functionalities are not available"
+     ]
+    }
+   ],
    "source": [
     "import os\n",
     "import pathlib\n",
@@ -52,7 +51,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": 3,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -62,7 +61,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": 4,
    "metadata": {},
    "outputs": [
     {
@@ -128,13 +127,13 @@
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "{'job_id': 'DirectRunner', 'metrics': {'rows': 6, 'good_images': 6, 'bad_images': None}, 'tfrecord_dir': '/tmp/tfrecords/tfrecorder-20201022-120722-create-tfrecords'}\n"
+      "{'job_id': 'DirectRunner', 'metrics': {'rows': 6, 'good_images': 6, 'bad_images': None}, 'tfrecord_dir': '/tmp/tfrecords/tfrecorder-20201027-173455-create-tfrecords'}\n"
      ]
     }
    ],
    "source": [
     "output_dir = pathlib.Path('/tmp/tfrecords')\n",
-    "results = tfrecorder.create_tfrecords(str(image_dir), output_dir)\n",
+    "results = tfrecorder.convert(str(image_dir), output_dir)\n",
     "print(results)"
    ]
   },
@@ -147,7 +146,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 9,
+   "execution_count": 5,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -156,7 +155,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 10,
+   "execution_count": 6,
    "metadata": {
     "pycharm": {
      "name": "#%%\n"
@@ -198,4 +197,4 @@
  },
  "nbformat": 4,
  "nbformat_minor": 4
-}
+}

samples/Convert-structured-data.ipynb

@@ -58,7 +58,7 @@
      "output_type": "stream",
      "text": [
       "Downloading data from https://web.stanford.edu/class/archive/cs/cs109/cs109.1166/stuff/titanic.csv\n",
-      "49152/44225 [=================================] - 0s 6us/step\n"
+      "49152/44225 [=================================] - 1s 12us/step\n"
      ]
     }
    ],
@@ -234,7 +234,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": 6,
    "metadata": {},
    "outputs": [
     {
@@ -298,7 +298,7 @@
     }
    ],
    "source": [
-    "results = tfrecorder.create_tfrecords(\n",
+    "results = tfrecorder.convert(\n",
     "    df, \n",
     "    './tfrecords', \n",
     "    schema=input_schema.Schema({\n",
@@ -317,14 +317,14 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": 7,
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "{'job_id': 'DirectRunner', 'metrics': {'rows': 887, 'good_images': None, 'bad_images': None}, 'tfrecord_dir': './tfrecords/tfrecorder-20201022-120403-create-tfrecords'}\n"
+      "{'job_id': 'DirectRunner', 'metrics': {'rows': 887, 'good_images': None, 'bad_images': None}, 'tfrecord_dir': './tfrecords/tfrecorder-20201027-173544-create-tfrecords'}\n"
      ]
     }
    ],
@@ -341,7 +341,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 9,
+   "execution_count": 8,
    "metadata": {},
    "outputs": [],
    "source": [
@@ -350,7 +350,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 10,
+   "execution_count": 9,
    "metadata": {},
    "outputs": [
     {

samples/Loading-a-TF-Dataset.ipynb

@@ -8,7 +8,7 @@
     "\n",
     "This notebook briefly demonstrates how to load a TF Dataset from TFRecord files generated by TFRecorder.\n",
     "Note that currently, the TFRecord files must be in a directory on your local machine.\n",
-    "The directory is expected to have the following structure, based on TFRecorder's `create_tfrecords` default output:\n",
+    "The directory is expected to have the following structure, based on TFRecorder's `convert` default output:\n",
     "```\n",
     "tfrecord_dir/\n",
     "    train-*.tfrecord.gz\n",
@@ -109,6 +109,13 @@
     "for d in train:\n",
     "  print(d['image_name'])"
    ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
   }
  ],
  "metadata": {

tfrecorder/__init__.py

@@ -15,7 +15,9 @@
 # limitations under the License.
 
 """Imports."""
+
 from tfrecorder import accessor
-from tfrecorder.client import create_tfrecords
-from tfrecorder.check import check_tfrecords
-from tfrecorder.dataset import load
+from tfrecorder.converter import convert
+from tfrecorder.dataset_loader import load
+from tfrecorder.converter import convert_and_load
+from tfrecorder.utils import inspect

tfrecorder/accessor.py

@@ -25,7 +25,7 @@
 import pandas as pd
 from IPython.core import display
 
-from tfrecorder import client
+from tfrecorder import converter
 from tfrecorder import constants
 from tfrecorder import input_schema
 
@@ -86,7 +86,7 @@ def to_tfr(
         display.HTML(
             '<b>Logging output to /tmp/{} </b>'.format(constants.LOGFILE)))
 
-    r = client.create_tfrecords(
+    r = converter.convert(
         self._df,
         output_dir=output_dir,
         schema=schema,

tfrecorder/cli.py

@@ -18,16 +18,16 @@
 
 import fire
 
-from tfrecorder import client
-from tfrecorder import check
+from tfrecorder import converter
+from tfrecorder import utils
 
 
 def main():
   """Entry point for command-line interface."""
 
   fire.Fire({
-      'create-tfrecords': client.create_tfrecords,
-      'check-tfrecords': check.check_tfrecords,
+      'convert': converter.convert,
+      'inspect': utils.inspect,
   })