Merge pull request #377 from hama1080/fix_data_loading

データ読み込み処理の修正
KAMONOHASHI · Jun 22, 2020 · 0c019a8 · 0c019a8
2 parents afef260 + cc4bf94
commit 0c019a8
Show file tree

Hide file tree

Showing 5 changed files with 34 additions and 21 deletions.
diff --git a/cli/kamonohashi/cli/data_set.py b/cli/kamonohashi/cli/data_set.py
@@ -188,7 +188,8 @@ def list_path_pairs(id):
     api = rest.DataSetApi(configuration.get_api_client())
     result = api.list_dataset_pathpairs(id)
     for x in result:
-      print(x.data_path + " " + x.stored_path)
+      print(repr(x.data_path)[1:-1])
+      print(repr(x.stored_path)[1:-1])
 
 @data_set.command('list-data-types')
 def list_data_types():

diff --git a/web-api/platypus/platypus/Migrations/20200615001935_v2.1.0.cs b/web-api/platypus/platypus/Migrations/20200615001935_v2.1.0.cs
@@ -35,19 +35,19 @@ protected override void Up(MigrationBuilder migrationBuilder)
                 name: "LocalDataSet",
                 table: "TrainingHistories",
                 nullable: false,
-                defaultValue: false);
+                defaultValue: true);
 
             migrationBuilder.AddColumn<bool>(
                 name: "LocalDataSet",
                 table: "NotebookHistories",
                 nullable: false,
-                defaultValue: false);
+                defaultValue: true);
 
             migrationBuilder.AddColumn<bool>(
                 name: "LocalDataSet",
                 table: "InferenceHistories",
                 nullable: false,
-                defaultValue: false);
+                defaultValue: true);
         }
 
         protected override void Down(MigrationBuilder migrationBuilder)

diff --git a/...atypus/platypus/ServiceModels/KubernetesModels/Templates/ConfigMap/inference_scripts.yaml b/...atypus/platypus/ServiceModels/KubernetesModels/Templates/ConfigMap/inference_scripts.yaml
@@ -17,13 +17,17 @@ data:
     fi
 
     # default: create symlink to raw-data, $LOCAL_DATASET = True : copy raw-data to local
-    prepare_cmd='ln -s "/kqi/raw/${stored_path}" "/kqi/input/${file_path}"'
+    prepare_cmd='ln -s "/kqi/raw/${stored_path}" "/kqi/input/${file_path_c_quote}"'
     if [ $LOCAL_DATASET = "True" ]; then
-        prepare_cmd='cp "/kqi/raw/${stored_path}" "/kqi/input/${file_path}"'
+        prepare_cmd='cp "/kqi/raw/${stored_path}" "/kqi/input/${file_path_c_quote}"'
     fi
-    while read file_path stored_path
+    while true
     do
-        mkdir -p "/kqi/input/$(dirname ${file_path})" && eval ${prepare_cmd}
+        read -r file_path
+        read -r stored_path
+        if [ -z "$stored_path" ] ; then break; fi
+        printf -v file_path_c_quote %b "$file_path"
+        mkdir -p /kqi/input/$(dirname "${file_path}") && eval ${prepare_cmd}
         if [ $? -ne 0 ]; then
             echo "create dataset failed"
             kqi inference halt $TRAINING_ID

diff --git a/...latypus/platypus/ServiceModels/KubernetesModels/Templates/ConfigMap/notebook_scripts.yaml b/...latypus/platypus/ServiceModels/KubernetesModels/Templates/ConfigMap/notebook_scripts.yaml
@@ -19,17 +19,21 @@ data:
             echo "dataset get failed"
         fi
         # default: create symlink to raw-data, $LOCAL_DATASET = True : copy raw-data to local
-        prepare_cmd='ln -s "/kqi/raw/${stored_path}" "/kqi/input/${file_path}"'
+        prepare_cmd='ln -s "/kqi/raw/${stored_path}" "/kqi/input/${file_path_c_quote}"'
         if [ $LOCAL_DATASET = "True" ]; then
-            prepare_cmd='cp "/kqi/raw/${stored_path}" "/kqi/input/${file_path}"'
+            prepare_cmd='cp "/kqi/raw/${stored_path}" "/kqi/input/${file_path_c_quote}"'
         fi
-        while read file_path stored_path
+        while true
         do
-            mkdir -p "/kqi/input/$(dirname ${file_path})" && eval ${prepare_cmd}
-        if [ $? -ne 0 ]; then
-            echo "create dataset failed"
-            exit 0
-        fi
+            read -r file_path
+            read -r stored_path
+            if [ -z "$stored_path" ] ; then break; fi
+            printf -v file_path_c_quote %b "$file_path"
+            mkdir -p /kqi/input/$(dirname "${file_path}") && eval ${prepare_cmd}
+            if [ $? -ne 0 ]; then
+                echo "create dataset failed"
+                exit 0
+            fi
         done </kqi/tmp/datapairs.txt
     fi
   main: |

diff --git a/...latypus/platypus/ServiceModels/KubernetesModels/Templates/ConfigMap/training_scripts.yaml b/...latypus/platypus/ServiceModels/KubernetesModels/Templates/ConfigMap/training_scripts.yaml
@@ -17,19 +17,23 @@ data:
     fi
 
     # default: create symlink to raw-data, $LOCAL_DATASET = True : copy raw-data to local
-    prepare_cmd='ln -s "/kqi/raw/${stored_path}" "/kqi/input/${file_path}"'
+    prepare_cmd='ln -s "/kqi/raw/${stored_path}" "/kqi/input/${file_path_c_quote}"'
     if [ $LOCAL_DATASET = "True" ]; then
-        prepare_cmd='cp "/kqi/raw/${stored_path}" "/kqi/input/${file_path}"'
+        prepare_cmd='cp "/kqi/raw/${stored_path}" "/kqi/input/${file_path_c_quote}"'
     fi
-    while read file_path stored_path
+    while true
     do
-        mkdir -p "/kqi/input/$(dirname ${file_path})" && eval ${prepare_cmd}
+        read -r file_path
+        read -r stored_path
+        if [ -z "$stored_path" ] ; then break; fi
+        printf -v file_path_c_quote %b "$file_path"
+        mkdir -p /kqi/input/$(dirname "${file_path}") && eval ${prepare_cmd}
         if [ $? -ne 0 ]; then
             echo "create dataset failed"
             kqi training halt $TRAINING_ID
             exit 0
         fi
-    done </kqi/tmp/datapairs.txt    
+    done </kqi/tmp/datapairs.txt
   main: |
     bash /kqi/scripts/common/wait-ready
     cd /kqi/git