Create an Intake Driver for Darknet

environment.yml

@@ -23,11 +23,15 @@ dependencies:
   - click-plugins
   - darknet
   - entrypoints
+  - fsspec
+  - intake
+  - numpy
   - pillow
 
   # Test Requirements (setup.py:test_requirements)
   - pytest >=3
   - pytest-cov
+  - pytest-mock
 
   # Documentation Requirements (setup.py:doc_requirements)
   - sphinx

examples/imagenet1k.ipynb

@@ -2,41 +2,41 @@
  "cells": [
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": 1,
    "metadata": {
     "collapsed": true
    },
    "outputs": [],
    "source": [
     "import os\n",
-    "import darknet\n",
-    "import pydarknet\n",
+    "import fsspec\n",
+    "\n",
+    "from PIL import Image as PILImage\n",
     "\n",
-    "from PIL import Image as PILImage"
+    "from darknet.py import ImageClassifier"
    ]
   },
   {
    "cell_type": "code",
    "execution_count": 2,
-   "outputs": [],
-   "source": [
-    "# Darknet is stupid... so all paths are relative at the moment.\n",
-    "os.chdir(\"..\")"
-   ],
    "metadata": {
-    "collapsed": false,
     "pycharm": {
      "name": "#%%\n"
     }
-   }
+   },
+   "outputs": [],
+   "source": [
+    "darknet_gh_url = \"github://pjreddie:darknet@master\""
+   ]
   },
   {
    "cell_type": "code",
    "execution_count": 3,
    "outputs": [],
    "source": [
     "# Load the ImageNet 1k labels/metadata\n",
-    "imagenet = darknet.Metadata(\"cfg/imagenet1k.data\")"
+    "with fsspec.open(f\"{darknet_gh_url}/data/imagenet.shortnames.list\", mode=\"rt\") as f:\n",
+    "    labels = [line.rstrip() for line in f.readlines()[:1000]]"
    ],
    "metadata": {
     "collapsed": false,
@@ -47,61 +47,59 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
-   "outputs": [],
-   "source": [
-    "# Load the Classifier\n",
-    "n = pydarknet.ImageClassifier(labels=imagenet.classes,\n",
-    "                              config_file=\"cfg/darknet53_448.cfg\",\n",
-    "                              weights_file=\"weights/darknet53_448.weights\")"
-   ],
+   "execution_count": 5,
    "metadata": {
-    "collapsed": false,
     "pycharm": {
      "name": "#%%\n"
     }
-   }
+   },
+   "outputs": [],
+   "source": [
+    "n = ImageClassifier(labels=labels,\n",
+    "                    config_url=f\"{darknet_gh_url}/cfg/darknet53_448.cfg\",\n",
+    "                    weights_url=\"https://pjreddie.com/media/files/darknet53_448.weights\")"
+   ]
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": 6,
+   "metadata": {
+    "pycharm": {
+     "name": "#%%\n"
+    }
+   },
    "outputs": [
     {
      "data": {
-      "text/plain": "[('malamute', 0.9832896),\n ('Eskimo dog', 0.0043422417),\n ('Siberian husky', 0.0032391667),\n ('Tibetan mastiff', 0.0030777778),\n ('Great Pyrenees', 0.0022481713)]"
+      "text/plain": "[('malamute', 0.98354006),\n ('Eskimo dog', 0.0042837244),\n ('Siberian husky', 0.0031863458),\n ('Tibetan mastiff', 0.0030448402),\n ('Great Pyrenees', 0.0022190544)]"
      },
-     "execution_count": 5,
+     "execution_count": 6,
      "metadata": {},
      "output_type": "execute_result"
     }
    ],
    "source": [
-    "img_filename = \"data/dog.jpg\"\n",
-    "n.classify(\"data/dog.jpg\", top=5)\n"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
+    "dog_url = f\"{darknet_gh_url}/data/dog.jpg\"\n",
+    "n.classify(dog_url, top=5)"
+   ]
   },
   {
    "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": 7,
    "outputs": [
     {
      "data": {
-      "text/plain": "[('malamute', 0.9832896),\n ('Eskimo dog', 0.0043422417),\n ('Siberian husky', 0.0032391667),\n ('Tibetan mastiff', 0.0030777778),\n ('Great Pyrenees', 0.0022481713)]"
+      "text/plain": "[('malamute', 0.98354006),\n ('Eskimo dog', 0.0042837244),\n ('Siberian husky', 0.0031863458),\n ('Tibetan mastiff', 0.0030448402),\n ('Great Pyrenees', 0.0022190544)]"
      },
-     "execution_count": 6,
+     "execution_count": 7,
      "metadata": {},
      "output_type": "execute_result"
     }
    ],
    "source": [
-    "pil_img = PILImage.open(img_filename)\n",
-    "n.classify(pil_img, top=5)"
+    "with fsspec.open(dog_url) as dog:\n",
+    "    res = n.classify(PILImage.open(dog), top=5)\n",
+    "res"
    ],
    "metadata": {
     "collapsed": false,
@@ -112,85 +110,90 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
-   "outputs": [],
+   "execution_count": 8,
+   "metadata": {
+    "pycharm": {
+     "name": "#%%\n"
+    }
+   },
+   "outputs": [
+    {
+     "data": {
+      "text/plain": "[('malamute', 0.98354006),\n ('Eskimo dog', 0.0042837244),\n ('Siberian husky', 0.0031863458),\n ('Tibetan mastiff', 0.0030448402),\n ('Great Pyrenees', 0.0022190544)]"
+     },
+     "execution_count": 8,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
    "source": [
     "try:\n",
     "    import cv2\n",
     "    # Note: Please note that cv2 nd-arrays are h*w*c ordered.\n",
-    "    cv2_img = cv2.imread(img_filename)\n",
+    "    with fsspec.open(dog_url) as dog:\n",
+    "        cv2_img = cv2.imread(dog.name)\n",
     "    cv2_img = cv2.cvtColor(cv2_img, cv2.COLOR_BGR2RGB)\n",
-    "    n.classify(cv2_img, top=5)\n",
+    "    res = n.classify(cv2_img, top=5)\n",
     "except ModuleNotFoundError:\n",
-    "    pass"
-   ],
+    "    pass\n",
+    "res"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
    "metadata": {
-    "collapsed": false,
     "pycharm": {
      "name": "#%%\n"
     }
-   }
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 8,
+   },
    "outputs": [
     {
      "data": {
-      "text/plain": "[('bald eagle', 0.55398),\n ('vulture', 0.21862003),\n ('kite', 0.19144633),\n ('ruddy turnstone', 0.004691909),\n ('ruffed grouse', 0.0033115256)]"
+      "text/plain": "[('bald eagle', 0.55666465),\n ('vulture', 0.21876547),\n ('kite', 0.18937683),\n ('ruddy turnstone', 0.004589723),\n ('ruffed grouse', 0.0032499917)]"
      },
-     "execution_count": 8,
+     "execution_count": 9,
      "metadata": {},
      "output_type": "execute_result"
     }
    ],
    "source": [
-    "n.classify(\"data/eagle.jpg\", top=5)"
-   ],
+    "n.classify(f\"{darknet_gh_url}/data/eagle.jpg\", top=5)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
    "metadata": {
-    "collapsed": false,
     "pycharm": {
      "name": "#%%\n"
     }
-   }
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 9,
+   },
    "outputs": [
     {
      "data": {
-      "text/plain": "[('electric guitar', 0.9876499),\n ('acoustic guitar', 0.009498796),\n ('banjo', 0.0011737668),\n ('pick', 0.00072627096),\n ('stage', 0.0005768967)]"
+      "text/plain": "[('electric guitar', 0.98759043),\n ('acoustic guitar', 0.009553942),\n ('banjo', 0.0011607071),\n ('pick', 0.0007309786),\n ('stage', 0.00058993115)]"
      },
-     "execution_count": 9,
+     "execution_count": 10,
      "metadata": {},
      "output_type": "execute_result"
     }
    ],
    "source": [
-    "import requests\n",
     "majesty_url=\"https://s3-us-west-2.amazonaws.com/static.music-man.com/website/images/instruments/instrument-77.png?1588624445\"\n",
-    "img = PILImage.open(requests.get(majesty_url, stream=True).raw)\n",
-    "n.classify(img, top=5)"
-   ],
-   "metadata": {
-    "collapsed": false,
-    "pycharm": {
-     "name": "#%%\n"
-    }
-   }
+    "n.classify(majesty_url, top=5)"
+   ]
   },
   {
    "cell_type": "code",
    "execution_count": null,
-   "outputs": [],
-   "source": [],
    "metadata": {
-    "collapsed": false,
     "pycharm": {
      "name": "#%%\n"
     }
-   }
+   },
+   "outputs": [],
+   "source": []
   }
  ],
  "metadata": {
@@ -202,16 +205,16 @@
   "language_info": {
    "codemirror_mode": {
     "name": "ipython",
-    "version": 2
+    "version": 3
    },
    "file_extension": ".py",
    "mimetype": "text/x-python",
    "name": "python",
    "nbconvert_exporter": "python",
-   "pygments_lexer": "ipython2",
-   "version": "2.7.6"
+   "pygments_lexer": "ipython3",
+   "version": "3.8.3"
   }
  },
  "nbformat": 4,
- "nbformat_minor": 0
+ "nbformat_minor": 1
 }

setup.py

@@ -14,6 +14,8 @@
     "click>=7.0",
     "click-plugins",
     "entrypoints",
+    "fsspec",
+    "intake",
     "numpy",
     "pillow",
     # fmt: on
@@ -33,6 +35,7 @@
     # fmt: off
     "pytest>=3",
     "pytest-cov",
+    "pytest-mock",
     # fmt: on
 ]
 
@@ -97,6 +100,12 @@
         "darknet.cli": [
             "py=darknet.py.cli:py",
         ],
+        "intake.drivers": [
+            "darknet = darknet.py.intake:DarknetSource"
+        ],
+        "intake.catalogs": [
+            "darknet = darknet.zoo:cat"
+        ]
     },
     # fmt: on
     ext_modules=ext_modules,

src/darknet/py/__init__.py

@@ -1,6 +1,5 @@
 """Top-level package for DarkNet OpenSource Neural Networks in Python."""
 from ._version import version as __version__  # noqa: F401
-
 from .network import Network
 from .classifier import Classifier, ImageClassifier
 from .detector import ImageDetector

src/darknet/py/classifier.py

@@ -10,8 +10,8 @@ class ClassifierBase(ABC):
     network: Network
     labels: list
 
-    def __init__(self, labels, config_file, weights_file, **kwargs):
-        self.network = Network(config_file, weights_file, **kwargs)
+    def __init__(self, labels, config_url, weights_url, **kwargs):
+        self.network = Network.open(config_url, weights_url, **kwargs)
 
         self.labels = range(self.network.output_size()) if labels is None else labels
         if len(self.labels) != self.network.output_size():

src/darknet/py/detector.py

@@ -8,8 +8,8 @@ class ImageDetector(object):
 
     _last_image_size = None
 
-    def __init__(self, labels, config_file, weights_file, **kwargs):
-        self.network = Network(config_file, weights_file, **kwargs)
+    def __init__(self, labels, config_url, weights_url, **kwargs):
+        self.network = Network.open(config_url, weights_url, **kwargs)
         self.labels = labels
 
     def detect(self, image, **kwargs):