Interactive API Experiment - Pytorch Re-ID on Market

openfl-tutorials/interactive_api/Pytorch_Market_Re-ID/director_folder/director_config.yaml

@@ -0,0 +1,4 @@
+settings:
+  listen_ip: localhost
+  sample_shape: ['64', '128', '3']
+  target_shape: ['2']

openfl-tutorials/interactive_api/Pytorch_Market_Re-ID/director_folder/start_director.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+set -e
+
+fx director start --disable-tls -c director_config.yaml

openfl-tutorials/interactive_api/Pytorch_Market_Re-ID/director_folder/start_director_with_tls.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+set -e
+FQDN=$1
+fx director start -c director_config.yaml -rc cert/root_ca.crt -pk cert/"${FQDN}".key -oc cert/"${FQDN}".crt

openfl-tutorials/interactive_api/Pytorch_Market_Re-ID/envoy_folder/market_shard_descriptor.py

@@ -0,0 +1,106 @@
+# Copyright (C) 2020-2021 Intel Corporation
+# SPDX-License-Identifier: Apache-2.0
+
+"""Market shard descriptor."""
+
+import re
+from logging import getLogger
+from pathlib import Path
+
+from PIL import Image
+
+from openfl.interface.interactive_api.shard_descriptor import ShardDescriptor
+
+
+logger = getLogger(__name__)
+
+# Previously download data and put to project folder
+# URL: https://www.kaggle.com/pengcw1/market-1501
+
+
+class MarketShardDescriptor(ShardDescriptor):
+    """
+    Market1501 Shard descriptor class.
+
+    Reference:
+    Zheng et al. Scalable Person Re-identification: A Benchmark. ICCV 2015.
+    URL: http://www.liangzheng.org/Project/project_reid.html
+
+    Dataset statistics:
+        identities: 1501 (+1 for background)
+        images: 12936 (train) + 3368 (query) + 15913 (gallery)
+    """
+
+    def __init__(self, datafolder: str = 'Market', rank_worldsize: str = '1,1') -> None:
+        """Initialize MarketShardDescriptor."""
+        super().__init__()
+
+        # Settings for sharding the dataset
+        self.rank, self.worldsize = tuple(int(num) for num in rank_worldsize.split(','))
+
+        self.pattern = re.compile(r'([-\d]+)_c(\d)')
+        # parent directory of project
+        self.dataset_dir = list(Path.cwd().parents[2].rglob(f'**/{datafolder}'))[0]
+        self.train_dir = self.dataset_dir / 'bounding_box_train'
+        self.query_dir = self.dataset_dir / 'query'
+        self.gal_dir = self.dataset_dir / 'bounding_box_test'
+
+        self._check_before_run()
+
+        self.train_path = list(self.train_dir.glob('*.jpg'))[self.rank - 1::self.worldsize]
+        self.query_path = list(self.query_dir.glob('*.jpg'))[self.rank - 1::self.worldsize]
+        self.gal_path = list(self.gal_dir.glob('*.jpg'))[self.rank - 1::self.worldsize]
+
+        self.mode = 'train'
+        self.imgs_path = self.train_path
+
+    def set_mode(self, mode='train'):
+        """Set mode for getitem."""
+        self.mode = mode
+        if self.mode == 'train':
+            self.imgs_path = self.train_path
+        elif self.mode == 'query':
+            self.imgs_path = self.query_path
+        elif self.mode == 'gallery':
+            self.imgs_path = self.gal_path
+        else:
+            raise Exception(f'Wrong mode: {mode}')
+
+    def __len__(self):
+        """Length of shard."""
+        return len(self.imgs_path)
+
+    def __getitem__(self, index: int):
+        """Return an item by the index."""
+        img_path = self.imgs_path[index]
+        pid, camid = map(int, self.pattern.search(img_path.name).groups())
+
+        img = Image.open(img_path)
+        return img, (pid, camid)
+
+    @property
+    def sample_shape(self):
+        """Return the sample shape info."""
+        return ['64', '128', '3']
+
+    @property
+    def target_shape(self):
+        """Return the target shape info."""
+        return ['2']
+
+    @property
+    def dataset_description(self) -> str:
+        """Return the dataset description."""
+        return (f'Market dataset, shard number {self.rank} '
+                f'out of {self.worldsize}')
+
+    def _check_before_run(self):
+        """Check if all files are available before going deeper."""
+        if not self.dataset_dir.exists():
+            raise RuntimeError(f'{self.dataset_dir} is not available')
+        if not self.train_dir.exists():
+            raise RuntimeError(f'{self.train_dir} is not available')
+        if not self.query_dir.exists():
+            raise RuntimeError(f'{self.query_dir} is not available')
+        if not self.gal_dir.exists():
+            raise RuntimeError(f'{self.gal_dir} is not available')

openfl-tutorials/interactive_api/Pytorch_Market_Re-ID/envoy_folder/shard_config_one.yaml

@@ -0,0 +1,4 @@
+template: market_shard_descriptor.MarketShardDescriptor
+params:
+  datafolder: Market
+  rank_worldsize: 1,2

openfl-tutorials/interactive_api/Pytorch_Market_Re-ID/envoy_folder/shard_config_two.yaml

@@ -0,0 +1,4 @@
+template: market_shard_descriptor.MarketShardDescriptor
+params:
+  datafolder: Market
+  rank_worldsize: 2,2

openfl-tutorials/interactive_api/Pytorch_Market_Re-ID/envoy_folder/start_envoy.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+set -e
+
+fx envoy start -n env_one --disable-tls -dh localhost -dp 50051

openfl-tutorials/interactive_api/Pytorch_Market_Re-ID/envoy_folder/start_envoy_with_tls.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+set -e
+ENVOY_NAME=$1
+DIRECTOR_FQDN=$2
+
+fx envoy start -n "$ENVOY_NAME" --shard-config-path shard_config.yaml -d "$DIRECTOR_FQDN":50051 -rc cert/root_ca.crt -pk cert/"$ENVOY_NAME".key -oc cert/"$ENVOY_NAME".crt