Model proto ll

.gitignore

@@ -5,3 +5,9 @@ __pycache__
 .vscode
 .ipynb_checkpoints
 venv/*
+
+*.jpg
+*.crt
+*.key
+*.pkl
+# openfl-tutorials/interactive_api_tutorials_experimental/director_kvasir

ll_prepare.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+set -e
+
+git clone https://github.com/igor-davidyuk/openfl.git
+cd openfl
+git checkout model-proto-ll
+python3.7 -m pip install --upgrade pip
+python3.7 -m pip install virtualenv
+python3.7 -m virtualenv --python=python3.7 env-openfl
+source ./env-openfl/bin/activate
+pip install -e .
+pip install torchvision
+cd openfl-tutorials/interactive_api_tutorials_experimental/director_kvasir/
+
+# jupyter notebook --ip=0.0.0.0 --no-browser --allow-root --NotebookApp.token='' --NotebookApp.password=''
+
+get best model "DONE"
+envoy registry - remove envoy 
+remove experiment workspace on envoy and director in case of fail (demo error) "DONE"
+directors API registry
+logs streaming "DONE"
+remove pickling
+shard descriptor `get_item` from validation set
+return error to API if collaborator fails
+passing envoy node info, choosing devices
+assigner
+send model
+adding tests
+workspace should be unpacked by CollaboratorManager component but not by the grpc client
+we do not need to deconstruct model proto on director side, just pass it to the aggregator
+add support for exporting pacages installed from github
+collaborator should try to connect the same fqdn the director is running on
+tensorboard falls after the first experimnet

openfl-tutorials/interactive_api_tutorials_(experimental)/VOC_LL_IA/VOC_shard_descriptor.py → openfl-tutorials/interactive_api_tutorials_experimental/VOC_LL_IA/VOC_shard_descriptor.py

@@ -48,12 +48,12 @@ class VOCDatasetSD(ShardDescriptor):
 
     def __init__(self, data_dir, split, keep_difficult=False):
         """Dataset for VOC data.
-
         Args:
             data_dir: the root of the VOC2007 or VOC2012 dataset,
                 the directory contains the following sub-directories:
                 Annotations, ImageSets, JPEGImages, SegmentationClass, SegmentationObject.
         """
+
         self.data_dir = data_dir
         self.split = split
         image_sets_file = os.path.join(self.data_dir, 'ImageSets', 'Main', f'{self.split}.txt')

openfl-tutorials/interactive_api_tutorials_(experimental)/director_kvasir/director_folder/start_director.py → openfl-tutorials/interactive_api_tutorials_experimental/director_kvasir/director_folder/start_director.py

@@ -7,20 +7,10 @@
 import asyncio
 import logging
 
-from rich.console import Console
-from rich.logging import RichHandler
-
 from openfl.component.director.director import serve
+from openfl.interface.cli import setup_logging
 
-root = logging.getLogger()
-root.setLevel(logging.INFO)
-console = Console(width=160)
-handler = RichHandler(console=console)
-formatter = logging.Formatter(
-    '[%(asctime)s][%(name)s][%(levelname)s] - %(message)s'
-)
-handler.setFormatter(formatter)
-root.addHandler(handler)
+setup_logging()
 
 logger = logging.getLogger(__name__)
 

openfl-tutorials/interactive_api_tutorials_(experimental)/director_kvasir/envoy_folder/kvasir_shard_descriptor.py → openfl-tutorials/interactive_api_tutorials_experimental/director_kvasir/envoy_folder/kvasir_shard_descriptor.py

@@ -23,7 +23,7 @@ def __init__(self, data_folder: str = 'kvasir_data',
         super().__init__()
 
         self.data_folder = Path.cwd() / data_folder
-        # self.download_data(self.data_folder)  # NOQA
+        self.download_data(self.data_folder)  # NOQA
 
         # Settings for resizing data
         self.enforce_image_hw = tuple(int(size) for size in enforce_image_hw.split(',')) if \
@@ -98,13 +98,11 @@ def dataset_description(self) -> str:
 
 
 if __name__ == '__main__':
-    import logging
-
-    root = logging.getLogger()
-    root.setLevel(logging.INFO)
+    from openfl.interface.cli import setup_logging
+    setup_logging()
 
     data_folder = 'data'
-    rank_worldsize = '1,2'
+    rank_worldsize = '1,100'
     enforce_image_hw = '529,622'
 
     kvasir_sd = KvasirShardDescriptor(

openfl-tutorials/interactive_api_tutorials_experimental/director_kvasir/envoy_folder/sd_requirements.txt

@@ -0,0 +1,2 @@
+numpy
+pillow